[YouTube] Potokens support implementation

checkstyle/checkstyle.xml

@@ -192,4 +192,9 @@
 
         <module name="SuppressWarningsHolder" />
     </module>
+
+    <!-- https://github.com/checkstyle/checkstyle/issues/11581 -->
+    <module name="SuppressionSingleFilter">
+        <property name="message" value="Unknown tag '(implNote|implSpec|apiNote)'\."/>
+    </module>
 </module>

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenProvider.java

@@ -0,0 +1,45 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import javax.annotation.Nullable;
+
+/**
+ * An interface to provide poTokens to YouTube player requests.
+ *
+ *
+ * <p>
+ * On some major clients, YouTube requires that the integrity of the device passes some checks to
+ * allow playback.
+ * </p>
+ *
+ * <p>
+ * These checks involve running codes to verify the integrity and using their result to generate a
+ * poToken (which likely stands for proof of origin token), using a visitor data ID for logged-out
+ * users.
+ * </p>
+ *
+ * <p>
+ * These tokens may have a role in triggering the sign in requirement.
+ * </p>
+ *
+ * @implNote This interface is expected to be thread-safe,
+ * as it may be accessed by multiple threads.
+ */
+public interface PoTokenProvider {
+
+    /**
+     * Get a {@link PoTokenResult} specific to the desktop website, a.k.a. the WEB InnerTube client.
+     *
+     * <p>
+     * To be generated and valid, poTokens from this client must be generated using Google's
+     * BotGuard machine, which requires a JavaScript engine with a good DOM implementation. They
+     * must be added to adaptive/DASH streaming URLs with the {@code pot} parameter.
+     * </p>
+     *
+     * @return a {@link PoTokenResult} specific to the WEB InnerTube client
+     */
+    @Nullable
+    PoTokenResult getWebClientPoToken();
+
+    @Nullable
+    PoTokenResult getAndroidClientPoToken();
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/PoTokenResult.java

@@ -0,0 +1,22 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import javax.annotation.Nonnull;
+import java.util.Objects;
+
+public final class PoTokenResult {
+
+    /**
+     * The visitor data associated with a poToken.
+     */
+    public final String visitorData;
+
+    /**
+     * The poToken, a Protobuf object encoded as a base 64 string.
+     */
+    public final String poToken;
+
+    public PoTokenResult(@Nonnull final String visitorData, @Nonnull final String poToken) {
+        this.visitorData = Objects.requireNonNull(visitorData);
+        this.poToken = Objects.requireNonNull(poToken);
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -1200,9 +1200,10 @@ public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
     @Nonnull
     public static JsonBuilder<JsonObject> prepareAndroidMobileJsonBuilder(
             @Nonnull final Localization localization,
-            @Nonnull final ContentCountry contentCountry) {
+            @Nonnull final ContentCountry contentCountry,
+            @Nullable final String visitorData) {
         // @formatter:off
-        return JsonObject.builder()
+        final JsonBuilder<JsonObject> builder = JsonObject.builder()
                 .object("context")
                     .object("client")
                         .value("clientName", "ANDROID")
@@ -1224,8 +1225,13 @@ public static JsonBuilder<JsonObject> prepareAndroidMobileJsonBuilder(
                         .value("androidSdkVersion", 34)
                         .value("hl", localization.getLocalizationCode())
                         .value("gl", contentCountry.getCountryCode())
-                        .value("utcOffsetMinutes", 0)
-                    .end()
+                        .value("utcOffsetMinutes", 0);
+
+        if (visitorData != null) {
+            builder.value("visitorData", visitorData);
+        }
+
+        builder.end()
                     .object("request")
                         .array("internalExperimentFlags")
                         .end()
@@ -1238,6 +1244,7 @@ public static JsonBuilder<JsonObject> prepareAndroidMobileJsonBuilder(
                     .end()
                 .end();
         // @formatter:on
+        return builder;
     }
 
     @Nonnull
@@ -1308,26 +1315,6 @@ public static JsonBuilder<JsonObject> prepareTvHtml5EmbedJsonBuilder(
         // @formatter:on
     }
 
-    @Nonnull
-    public static JsonObject getWebPlayerResponse(
-            @Nonnull final Localization localization,
-            @Nonnull final ContentCountry contentCountry,
-            @Nonnull final String videoId) throws IOException, ExtractionException {
-        final byte[] body = JsonWriter.string(
-                        prepareDesktopJsonBuilder(localization, contentCountry)
-                                .value(VIDEO_ID, videoId)
-                                .value(CONTENT_CHECK_OK, true)
-                                .value(RACY_CHECK_OK, true)
-                                .done())
-                .getBytes(StandardCharsets.UTF_8);
-        final String url = YOUTUBEI_V1_URL + "player" + "?" + DISABLE_PRETTY_PRINT_PARAMETER
-                + "&$fields=microformat,playabilityStatus,storyboards,videoDetails";
-
-        return JsonUtils.toJsonObject(getValidJsonResponseBody(
-                getDownloader().postWithContentTypeJson(
-                        url, getYouTubeHeaders(), body, localization)));
-    }
-
     @Nonnull
     public static byte[] createTvHtml5EmbedPlayerBody(
             @Nonnull final Localization localization,

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamHelper.java

@@ -0,0 +1,165 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonWriter;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.localization.ContentCountry;
+import org.schabi.newpipe.extractor.localization.Localization;
+import org.schabi.newpipe.extractor.utils.JsonUtils;
+
+import javax.annotation.Nonnull;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CONTENT_CHECK_OK;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CPN;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.RACY_CHECK_OK;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.VIDEO_ID;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateTParameter;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonAndroidPostResponse;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonIosPostResponse;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYouTubeHeaders;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareAndroidMobileJsonBuilder;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareIosMobileJsonBuilder;
+
+public final class YoutubeStreamHelper {
+
+    private static final String STREAMING_DATA = "streamingData";
+    private static final String PLAYER = "player";
+    private static final String SERVICE_INTEGRITY_DIMENSIONS = "serviceIntegrityDimensions";
+    private static final String PO_TOKEN = "poToken";
+
+    private YoutubeStreamHelper() {
+    }
+
+    @Nonnull
+    public static JsonObject getWebMetadataPlayerResponse(
+            @Nonnull final Localization localization,
+            @Nonnull final ContentCountry contentCountry,
+            @Nonnull final String videoId) throws IOException, ExtractionException {
+        final byte[] body = JsonWriter.string(
+                        prepareDesktopJsonBuilder(localization, contentCountry)
+                                .value(VIDEO_ID, videoId)
+                                .value(CONTENT_CHECK_OK, true)
+                                .value(RACY_CHECK_OK, true)
+                                .done())
+                .getBytes(StandardCharsets.UTF_8);
+        final String url = YOUTUBEI_V1_URL + "player" + "?" + DISABLE_PRETTY_PRINT_PARAMETER
+                + "&$fields=microformat,playabilityStatus,storyboards,videoDetails";
+
+        return JsonUtils.toJsonObject(getValidJsonResponseBody(
+                getDownloader().postWithContentTypeJson(
+                        url, getYouTubeHeaders(), body, localization)));
+    }
+
+    @Nonnull
+    public static JsonObject getWebFullPlayerResponse(
+            @Nonnull final Localization localization,
+            @Nonnull final ContentCountry contentCountry,
+            @Nonnull final String videoId,
+            @Nonnull final PoTokenResult webPoTokenResult) throws IOException, ExtractionException {
+        final byte[] body = JsonWriter.string(
+                        prepareDesktopJsonBuilder(
+                                localization,
+                                contentCountry,
+                                webPoTokenResult.visitorData
+                        )
+                                .value(VIDEO_ID, videoId)
+                                .value(CONTENT_CHECK_OK, true)
+                                .value(RACY_CHECK_OK, true)
+                                .object(SERVICE_INTEGRITY_DIMENSIONS)
+                                .value(PO_TOKEN, webPoTokenResult.poToken)
+                                .end()
+                                .done())
+                .getBytes(StandardCharsets.UTF_8);
+        final String url = YOUTUBEI_V1_URL + "player" + "?" + DISABLE_PRETTY_PRINT_PARAMETER;
+
+        return JsonUtils.toJsonObject(getValidJsonResponseBody(
+                getDownloader().postWithContentTypeJson(
+                        url, getYouTubeHeaders(), body, localization)));
+    }
+
+    public static JsonObject getAndroidPlayerResponse(
+            @Nonnull final ContentCountry contentCountry,
+            @Nonnull final Localization localization,
+            @Nonnull final String videoId,
+            @Nonnull final String androidCpn,
+            @Nonnull final PoTokenResult androidPoTokenResult
+    )
+            throws IOException, ExtractionException {
+        final byte[] mobileBody = JsonWriter.string(
+                        prepareAndroidMobileJsonBuilder(
+                                localization,
+                                contentCountry,
+                                androidPoTokenResult.visitorData
+                        )
+                                .value(VIDEO_ID, videoId)
+                                .value(CPN, androidCpn)
+                                .value(CONTENT_CHECK_OK, true)
+                                .value(RACY_CHECK_OK, true)
+                                .object(SERVICE_INTEGRITY_DIMENSIONS)
+                                .value(PO_TOKEN, androidPoTokenResult.poToken)
+                                .end()
+                                .done())
+                .getBytes(StandardCharsets.UTF_8);
+
+        return getJsonAndroidPostResponse(
+                "player",
+                mobileBody,
+                localization,
+                "&t=" + generateTParameter() + "&id=" + videoId);
+    }
+
+    public static JsonObject getAndroidReelPlayerResponse(
+            @Nonnull final ContentCountry contentCountry,
+            @Nonnull final Localization localization,
+            @Nonnull final String videoId,
+            @Nonnull final String androidCpn
+    )
+            throws IOException, ExtractionException {
+        final byte[] mobileBody = JsonWriter.string(
+                        prepareAndroidMobileJsonBuilder(localization, contentCountry, null)
+                                .object("playerRequest")
+                                .value(VIDEO_ID, videoId)
+                                .end()
+                                .value("disablePlayerResponse", false)
+                                .value(VIDEO_ID, videoId)
+                                .value(CPN, androidCpn)
+                                .value(CONTENT_CHECK_OK, true)
+                                .value(RACY_CHECK_OK, true)
+                                .done())
+                .getBytes(StandardCharsets.UTF_8);
+
+        final JsonObject androidPlayerResponse = getJsonAndroidPostResponse(
+                "reel/reel_item_watch",
+                mobileBody,
+                localization,
+                "&t=" + generateTParameter() + "&id=" + videoId + "&$fields=playerResponse");
+
+        return androidPlayerResponse.getObject("playerResponse");
+    }
+
+    public static JsonObject getIosPlayerResponse(@Nonnull final ContentCountry contentCountry,
+                                                  @Nonnull final Localization localization,
+                                                  @Nonnull final String videoId,
+                                                  @Nonnull final String iosCpn)
+            throws IOException, ExtractionException {
+        final byte[] mobileBody = JsonWriter.string(
+                        prepareIosMobileJsonBuilder(localization, contentCountry)
+                                .value(VIDEO_ID, videoId)
+                                .value(CPN, iosCpn)
+                                .value(CONTENT_CHECK_OK, true)
+                                .value(RACY_CHECK_OK, true)
+                                .done())
+                .getBytes(StandardCharsets.UTF_8);
+
+        return getJsonIosPostResponse(PLAYER,
+                mobileBody, localization, "&t=" + generateTParameter()
+                        + "&id=" + videoId);
+    }
+}