Skip to content

Commit e747a89

Browse files
TobiGrTobiGr
authored
Merge pull request #600 from TeamNewPipe/youtube-eu-cosent
[YouTube] Set EU consent cookie
2 parents e61ceef + 33173eb commit e747a89

File tree

190 files changed

+5000
-5769
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

190 files changed

+5000
-5769
lines changed

‎extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Request.java

+3-5
Original file line numberDiff line numberDiff line change
@@ -123,12 +123,10 @@ public Builder url(String url) {
123123
* Any default headers that the implementation may have, <b>should</b> be overridden by these.
124124
*/
125125
public Builder headers(@Nullable Map<String, List<String>> headers) {
126-
if (headers == null) {
127-
this.headers.clear();
128-
return this;
129-
}
130126
this.headers.clear();
131-
this.headers.putAll(headers);
127+
if (headers != null) {
128+
this.headers.putAll(headers);
129+
}
132130
return this;
133131
}
134132

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

+84-29
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,7 @@
2828
import java.time.OffsetDateTime;
2929
import java.time.ZoneOffset;
3030
import java.time.format.DateTimeParseException;
31-
import java.util.ArrayList;
32-
import java.util.Collections;
33-
import java.util.HashMap;
34-
import java.util.List;
35-
import java.util.Map;
36-
import java.util.Objects;
31+
import java.util.*;
3732

3833
import javax.annotation.Nonnull;
3934
import javax.annotation.Nullable;
@@ -79,6 +74,19 @@ private YoutubeParsingHelper() {
7974
private static final String[] HARDCODED_YOUTUBE_MUSIC_KEYS = {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "0.1"};
8075
private static String[] youtubeMusicKeys;
8176

77+
private static Random numberGenerator = new Random();
78+
79+
/**
80+
* <code>PENDING+</code> means that the user did not yet submit their choices.
81+
* Therefore, YouTube & Google should not track the user, because they did not give consent.
82+
* The three digits at the end can be random, but are required.
83+
*/
84+
private static final String CONSENT_COOKIE_VALUE = "PENDING+";
85+
/**
86+
* Youtube <code>CONSENT</code> cookie. Should prevent redirect to consent.youtube.com
87+
*/
88+
private static final String CONSENT_COOKIE = "CONSENT=" + CONSENT_COOKIE_VALUE;
89+
8290
private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id=";
8391
private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
8492

@@ -388,6 +396,15 @@ public static void resetClientVersionAndKey() {
388396
key = null;
389397
}
390398

399+
/**
400+
* <p>
401+
* <b>Only use in tests.</b>
402+
* </p>
403+
*/
404+
public static void setNumberGenerator(Random random) {
405+
numberGenerator = random;
406+
}
407+
391408
public static boolean areHardcodedYoutubeMusicKeysValid() throws IOException, ReCaptchaException {
392409
final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + HARDCODED_YOUTUBE_MUSIC_KEYS[0];
393410

@@ -427,6 +444,7 @@ public static boolean areHardcodedYoutubeMusicKeysValid() throws IOException, Re
427444
headers.put("Origin", Collections.singletonList("https://music.youtube.com"));
428445
headers.put("Referer", Collections.singletonList("music.youtube.com"));
429446
headers.put("Content-Type", Collections.singletonList("application/json"));
447+
addCookieHeader(headers);
430448

431449
final String response = getDownloader().post(url, headers, json).responseBody();
432450

@@ -629,34 +647,19 @@ public static String getValidJsonResponseBody(final Response response)
629647
public static Response getResponse(final String url, final Localization localization)
630648
throws IOException, ExtractionException {
631649
final Map<String, List<String>> headers = new HashMap<>();
632-
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
633-
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
650+
addYouTubeHeaders(headers);
634651

635652
final Response response = getDownloader().get(url, headers, localization);
636653
getValidJsonResponseBody(response);
637654

638655
return response;
639656
}
640657

641-
public static String extractCookieValue(final String cookieName, final Response response) {
642-
final List<String> cookies = response.responseHeaders().get("set-cookie");
643-
int startIndex;
644-
String result = "";
645-
for (final String cookie : cookies) {
646-
startIndex = cookie.indexOf(cookieName);
647-
if (startIndex != -1) {
648-
result = cookie.substring(startIndex + cookieName.length() + "=".length(),
649-
cookie.indexOf(";", startIndex));
650-
}
651-
}
652-
return result;
653-
}
654-
655658
public static JsonArray getJsonResponse(final String url, final Localization localization)
656659
throws IOException, ExtractionException {
657660
Map<String, List<String>> headers = new HashMap<>();
658-
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
659-
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
661+
addYouTubeHeaders(headers);
662+
660663
final Response response = getDownloader().get(url, headers, localization);
661664

662665
return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
@@ -665,17 +668,69 @@ public static JsonArray getJsonResponse(final String url, final Localization loc
665668
public static JsonArray getJsonResponse(final Page page, final Localization localization)
666669
throws IOException, ExtractionException {
667670
final Map<String, List<String>> headers = new HashMap<>();
668-
if (!isNullOrEmpty(page.getCookies())) {
669-
headers.put("Cookie", Collections.singletonList(join(";", "=", page.getCookies())));
670-
}
671-
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
672-
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
671+
addYouTubeHeaders(headers);
673672

674673
final Response response = getDownloader().get(page.getUrl(), headers, localization);
675674

676675
return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
677676
}
678677

678+
/**
679+
* Add required headers and cookies to an existing headers Map.
680+
* @see #addClientInfoHeaders(Map)
681+
* @see #addCookieHeader(Map)
682+
*/
683+
public static void addYouTubeHeaders(final Map<String, List<String>> headers)
684+
throws IOException, ExtractionException {
685+
addClientInfoHeaders(headers);
686+
addCookieHeader(headers);
687+
}
688+
689+
/**
690+
* Add the <code>X-YouTube-Client-Name</code> and <code>X-YouTube-Client-Version</code> headers.
691+
* @param headers The headers which should be completed
692+
*/
693+
public static void addClientInfoHeaders(final Map<String, List<String>> headers)
694+
throws IOException, ExtractionException {
695+
if (headers.get("X-YouTube-Client-Name") == null) {
696+
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
697+
}
698+
if (headers.get("X-YouTube-Client-Version") == null) {
699+
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
700+
}
701+
}
702+
703+
/**
704+
* Add the <code>CONSENT</code> cookie to prevent redirect to <code>consent.youtube.com</code>
705+
* @see #CONSENT_COOKIE
706+
* @param headers the headers which should be completed
707+
*/
708+
public static void addCookieHeader(final Map<String, List<String>> headers) {
709+
if (headers.get("Cookie") == null) {
710+
headers.put("Cookie", Arrays.asList(generateConsentCookie()));
711+
} else {
712+
headers.get("Cookie").add(generateConsentCookie());
713+
}
714+
}
715+
716+
public static String generateConsentCookie() {
717+
return CONSENT_COOKIE + 100 + numberGenerator.nextInt(900);
718+
}
719+
720+
public static String extractCookieValue(final String cookieName, final Response response) {
721+
final List<String> cookies = response.responseHeaders().get("set-cookie");
722+
int startIndex;
723+
String result = "";
724+
for (final String cookie : cookies) {
725+
startIndex = cookie.indexOf(cookieName);
726+
if (startIndex != -1) {
727+
result = cookie.substring(startIndex + cookieName.length() + "=".length(),
728+
cookie.indexOf(";", startIndex));
729+
}
730+
}
731+
return result;
732+
}
733+
679734
/**
680735
* Shared alert detection function, multiple endpoints return the error similarly structured.
681736
* <p>

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java

+7-2
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,9 @@
2020

2121
import java.io.IOException;
2222
import java.util.Collections;
23+
import java.util.HashMap;
2324
import java.util.List;
25+
import java.util.Map;
2426

2527
import javax.annotation.Nonnull;
2628
import javax.annotation.Nullable;
@@ -130,8 +132,11 @@ public long getStreamCount() {
130132
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
131133
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
132134
collectStreamsFrom(collector, playlistData.getArray("contents"));
133-
return new InfoItemsPage<>(collector,
134-
new Page(getNextPageUrlFrom(playlistData), Collections.singletonMap(COOKIE_NAME, cookieValue)));
135+
136+
final Map<String, String> cookies = new HashMap<>();
137+
cookies.put(COOKIE_NAME, cookieValue);
138+
139+
return new InfoItemsPage<>(collector, new Page(getNextPageUrlFrom(playlistData), cookies));
135140
}
136141

137142
private String getNextPageUrlFrom(final JsonObject playlistJson) throws ExtractionException {

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java

+9-6
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@
1212

1313
import java.io.IOException;
1414
import java.net.URLEncoder;
15-
import java.util.ArrayList;
16-
import java.util.List;
15+
import java.util.*;
1716

17+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addCookieHeader;
1818
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
1919

2020
/*
@@ -45,17 +45,20 @@ public YoutubeSuggestionExtractor(StreamingService service) {
4545

4646
@Override
4747
public List<String> suggestionList(String query) throws IOException, ExtractionException {
48-
Downloader dl = NewPipe.getDownloader();
49-
List<String> suggestions = new ArrayList<>();
48+
final Downloader dl = NewPipe.getDownloader();
49+
final List<String> suggestions = new ArrayList<>();
5050

51-
String url = "https://suggestqueries.google.com/complete/search"
51+
final String url = "https://suggestqueries.google.com/complete/search"
5252
+ "?client=" + "youtube" //"firefox" for JSON, 'toolbar' for xml
5353
+ "&jsonp=" + "JP"
5454
+ "&ds=" + "yt"
5555
+ "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(), UTF_8)
5656
+ "&q=" + URLEncoder.encode(query, UTF_8);
5757

58-
String response = dl.get(url, getExtractorLocalization()).responseBody();
58+
final Map<String, List<String>> headers = new HashMap<>();
59+
addCookieHeader(headers);
60+
61+
String response = dl.get(url, headers, getExtractorLocalization()).responseBody();
5962
// trim JSONP part "JP(...)"
6063
response = response.substring(3, response.length() - 1);
6164
try {

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java

+8
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
1414

1515
import java.io.IOException;
16+
import java.util.Random;
1617

1718
import static org.hamcrest.CoreMatchers.containsString;
1819
import static org.hamcrest.MatcherAssert.assertThat;
@@ -32,6 +33,7 @@ public static class NotAvailable {
3233
@BeforeClass
3334
public static void setUp() throws IOException {
3435
YoutubeParsingHelper.resetClientVersionAndKey();
36+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
3537
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "notAvailable"));
3638
}
3739

@@ -54,6 +56,7 @@ public static class NotSupported {
5456
@BeforeClass
5557
public static void setUp() throws IOException {
5658
YoutubeParsingHelper.resetClientVersionAndKey();
59+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
5760
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "notSupported"));
5861
}
5962

@@ -71,6 +74,7 @@ public static class Gronkh implements BaseChannelExtractorTest {
7174
@BeforeClass
7275
public static void setUp() throws Exception {
7376
YoutubeParsingHelper.resetClientVersionAndKey();
77+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
7478
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "gronkh"));
7579
extractor = (YoutubeChannelExtractor) YouTube
7680
.getChannelExtractor("http://www.youtube.com/user/Gronkh");
@@ -168,6 +172,7 @@ public static class VSauce implements BaseChannelExtractorTest {
168172
@BeforeClass
169173
public static void setUp() throws Exception {
170174
YoutubeParsingHelper.resetClientVersionAndKey();
175+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
171176
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "VSauce"));
172177
extractor = (YoutubeChannelExtractor) YouTube
173178
.getChannelExtractor("https://www.youtube.com/user/Vsauce");
@@ -265,6 +270,7 @@ public static class Kurzgesagt implements BaseChannelExtractorTest {
265270
@BeforeClass
266271
public static void setUp() throws Exception {
267272
YoutubeParsingHelper.resetClientVersionAndKey();
273+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
268274
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "kurzgesagt"));
269275
extractor = (YoutubeChannelExtractor) YouTube
270276
.getChannelExtractor("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q");
@@ -383,6 +389,7 @@ public static class CaptainDisillusion implements BaseChannelExtractorTest {
383389
@BeforeClass
384390
public static void setUp() throws Exception {
385391
YoutubeParsingHelper.resetClientVersionAndKey();
392+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
386393
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "captainDisillusion"));
387394
extractor = (YoutubeChannelExtractor) YouTube
388395
.getChannelExtractor("https://www.youtube.com/user/CaptainDisillusion/videos");
@@ -478,6 +485,7 @@ public static class RandomChannel implements BaseChannelExtractorTest {
478485
@BeforeClass
479486
public static void setUp() throws Exception {
480487
YoutubeParsingHelper.resetClientVersionAndKey();
488+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
481489
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "random"));
482490
extractor = (YoutubeChannelExtractor) YouTube
483491
.getChannelExtractor("https://www.youtube.com/channel/UCUaQMQS9lY5lit3vurpXQ6w");

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java

+2
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import java.util.LinkedHashMap;
1515
import java.util.List;
1616
import java.util.Map;
17+
import java.util.Random;
1718

1819
import static org.junit.Assert.fail;
1920
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@@ -30,6 +31,7 @@ public class YoutubeChannelLocalizationTest {
3031
@Test
3132
public void testAllSupportedLocalizations() throws Exception {
3233
YoutubeParsingHelper.resetClientVersionAndKey();
34+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
3335
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "localization"));
3436

3537
testLocalizationsFor("https://www.youtube.com/user/NBCNews");

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java

+5
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
import java.io.IOException;
1717
import java.util.List;
18+
import java.util.Random;
1819

1920
import static org.junit.Assert.assertEquals;
2021
import static org.junit.Assert.assertFalse;
@@ -36,6 +37,7 @@ public static class Thomas {
3637
@BeforeClass
3738
public static void setUp() throws Exception {
3839
YoutubeParsingHelper.resetClientVersionAndKey();
40+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
3941
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "thomas"));
4042
extractor = (YoutubeCommentsExtractor) YouTube
4143
.getCommentsExtractor(url);
@@ -124,6 +126,7 @@ public static class EmptyComment {
124126
@BeforeClass
125127
public static void setUp() throws Exception {
126128
YoutubeParsingHelper.resetClientVersionAndKey();
129+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
127130
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "empty"));
128131
extractor = (YoutubeCommentsExtractor) YouTube
129132
.getCommentsExtractor(url);
@@ -163,6 +166,7 @@ public static class HeartedByCreator {
163166
@BeforeClass
164167
public static void setUp() throws Exception {
165168
YoutubeParsingHelper.resetClientVersionAndKey();
169+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
166170
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "hearted"));
167171
extractor = (YoutubeCommentsExtractor) YouTube
168172
.getCommentsExtractor(url);
@@ -205,6 +209,7 @@ public static class Pinned {
205209
@BeforeClass
206210
public static void setUp() throws Exception {
207211
YoutubeParsingHelper.resetClientVersionAndKey();
212+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
208213
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "pinned"));
209214
extractor = (YoutubeCommentsExtractor) YouTube
210215
.getCommentsExtractor(url);

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java

+3
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
99
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor;
1010

11+
import java.util.Random;
12+
1113
import static org.junit.Assert.assertEquals;
1214
import static org.junit.Assert.assertTrue;
1315
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@@ -24,6 +26,7 @@ public static class Kurzgesagt implements BaseListExtractorTest {
2426
@BeforeClass
2527
public static void setUp() throws Exception {
2628
YoutubeParsingHelper.resetClientVersionAndKey();
29+
YoutubeParsingHelper.setNumberGenerator(new Random(1));
2730
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH));
2831
extractor = (YoutubeFeedExtractor) YouTube
2932
.getFeedExtractor("https://www.youtube.com/user/Kurzgesagt");

0 commit comments

Comments
 (0)