Merge branch 'dev_newpipe'

MaintainTeam · Jul 24, 2024 · 0b1008f · 0b1008f
2 parents 659af08 + 2d36945
commit 0b1008f
Show file tree

Hide file tree

Showing 421 changed files with 30,736 additions and 28,994 deletions.
diff --git a/.github/README.md b/.github/README.md
diff --git a/...va/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java b/...va/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java
@@ -16,6 +16,7 @@
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
 import org.schabi.newpipe.extractor.utils.ImageSuffix;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -155,25 +156,34 @@ public static String getImageUrl(final long id, final boolean isAlbum) {
 
     /**
      * @return <code>true</code> if the given URL looks like it comes from a bandcamp custom domain
-     * or if it comes from <code>bandcamp.com</code> itself
+     * or a <code>*.bandcamp.com</code> subdomain
      */
-    public static boolean isSupportedDomain(final String url) throws ParsingException {
+    public static boolean isArtistDomain(final String url) throws ParsingException {
 
         // Accept all bandcamp.com URLs
         if (url.toLowerCase().matches("https?://.+\\.bandcamp\\.com(/.*)?")) {
             return true;
         }
 
+        // Reject non-artist bandcamp.com URLs
+        if (url.toLowerCase().matches("https?://bandcamp\\.com(/.*)?")) {
+            return false;
+        }
+
         try {
             // Test other URLs for whether they contain a footer that links to bandcamp
-            return Jsoup.parse(NewPipe.getDownloader().get(url).responseBody())
-                    .getElementById("pgFt")
-                    .getElementById("pgFt-inner")
-                    .getElementById("footer-logo-wrapper")
-                    .getElementById("footer-logo")
-                    .getElementsByClass("hiddenAccess")
-                    .text().equals("Bandcamp");
-        } catch (final NullPointerException e) {
+            return Jsoup.parse(
+                    NewPipe.getDownloader()
+                            .get(Utils.replaceHttpWithHttps(url))
+                            .responseBody()
+                    )
+                    .getElementsByClass("cart-wrapper")
+                    .get(0)
+                    .getElementsByTag("a")
+                    .get(0)
+                    .attr("href")
+                    .equals("https://bandcamp.com/cart");
+        } catch (final NullPointerException | IndexOutOfBoundsException e) {
             return false;
         } catch (final IOException | ReCaptchaException e) {
             throw new ParsingException("Could not determine whether URL is custom domain "

diff --git a/...ava/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java b/...ava/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioExtractor.java
@@ -24,7 +24,7 @@
 public class BandcampRadioExtractor extends KioskExtractor<StreamInfoItem> {
 
     public static final String KIOSK_RADIO = "Radio";
-    public static final String RADIO_API_URL = BASE_API_URL + "/bcweekly/1/list";
+    public static final String RADIO_API_URL = BASE_API_URL + "/bcweekly/3/list";
 
     private JsonObject json = null;
 

diff --git a/...schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java b/...schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioInfoItemExtractor.java
@@ -35,6 +35,12 @@ public long getDuration() {
         return 0;
     }
 
+    @Nullable
+    @Override
+    public String getShortDescription() {
+        return show.getString("desc");
+    }
+
     @Nullable
     @Override
     public String getTextualUploadDate() {
@@ -75,8 +81,8 @@ public long getViewCount() {
 
     @Override
     public String getUploaderName() {
-        // JSON does not contain uploader name
-        return "";
+        // The "title" field contains the title of the series, e.g. "Bandcamp Weekly".
+        return show.getString("title");
     }
 
     @Override

diff --git a/.../services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java b/.../services/bandcamp/extractors/streaminfoitem/BandcampPlaylistStreamInfoItemExtractor.java
@@ -43,7 +43,12 @@ public String getName() {
 
     @Override
     public String getUrl() {
-        return getUploaderUrl() + track.getString("title_link");
+        final String relativeUrl = track.getString("title_link");
+        if (relativeUrl != null) {
+            return getUploaderUrl() + relativeUrl;
+        } else {
+            return null;
+        }
     }
 
     @Override
@@ -66,7 +71,7 @@ public String getUploaderName() {
     @Nonnull
     @Override
     public List<Image> getThumbnails() throws ParsingException {
-        if (substituteCovers.isEmpty()) {
+        if (substituteCovers.isEmpty() && getUrl() != null) {
             try {
                 final StreamExtractor extractor = service.getStreamExtractor(getUrl());
                 extractor.fetchPage();

diff --git a/...bi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java b/...bi/newpipe/extractor/services/bandcamp/linkHandler/BandcampChannelLinkHandlerFactory.java
@@ -37,7 +37,8 @@ public static BandcampChannelLinkHandlerFactory getInstance() {
     @Override
     public String getId(final String url) throws ParsingException, UnsupportedOperationException {
         try {
-            final String response = NewPipe.getDownloader().get(url).responseBody();
+            final String response = NewPipe.getDownloader().get(Utils.replaceHttpWithHttps(url))
+                    .responseBody();
 
             // Use band data embedded in website to extract ID
             final JsonObject bandData = JsonUtils.getJsonData(response, "data-band");
@@ -96,7 +97,7 @@ public boolean onAcceptUrl(final String url) throws ParsingException {
             }
 
             // Test whether domain is supported
-            return BandcampExtractorHelper.isSupportedDomain(lowercaseUrl);
+            return BandcampExtractorHelper.isArtistDomain(lowercaseUrl);
         }
     }
 }
diff --git a/...i/newpipe/extractor/services/bandcamp/linkHandler/BandcampCommentsLinkHandlerFactory.java b/...i/newpipe/extractor/services/bandcamp/linkHandler/BandcampCommentsLinkHandlerFactory.java
@@ -5,6 +5,7 @@
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
 import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.util.List;
 
@@ -29,7 +30,7 @@ public static BandcampCommentsLinkHandlerFactory getInstance() {
 
     @Override
     public String getId(final String url) throws ParsingException, UnsupportedOperationException {
-        return url;
+        return Utils.replaceHttpWithHttps(url);
     }
 
     @Override
@@ -44,14 +45,14 @@ public boolean onAcceptUrl(final String url) throws ParsingException {
         }
 
         // Test whether domain is supported
-        return BandcampExtractorHelper.isSupportedDomain(url);
+        return BandcampExtractorHelper.isArtistDomain(url);
     }
 
     @Override
     public String getUrl(final String id,
                          @Nonnull final List<FilterItem> contentFilter,
                          @Nullable final List<FilterItem> sortFilter)
             throws ParsingException, UnsupportedOperationException {
-        return id;
+        return Utils.replaceHttpWithHttps(id);
     }
 }
diff --git a/...i/newpipe/extractor/services/bandcamp/linkHandler/BandcampPlaylistLinkHandlerFactory.java b/...i/newpipe/extractor/services/bandcamp/linkHandler/BandcampPlaylistLinkHandlerFactory.java
@@ -7,6 +7,7 @@
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
 import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.util.List;
 
@@ -38,7 +39,7 @@ public String getUrl(final String url,
                          @Nonnull final List<FilterItem> contentFilter,
                          @Nullable final List<FilterItem> sortFilter)
             throws ParsingException, UnsupportedOperationException {
-        return url;
+        return Utils.replaceHttpWithHttps(url);
     }
 
     /**
@@ -53,6 +54,6 @@ public boolean onAcceptUrl(final String url) throws ParsingException {
         }
 
         // Test whether domain is supported
-        return BandcampExtractorHelper.isSupportedDomain(url);
+        return BandcampExtractorHelper.isArtistDomain(url);
     }
 }
diff --git a/...bi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java b/...bi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java
@@ -10,7 +10,6 @@
 import org.schabi.newpipe.extractor.services.bandcamp.search.filter.BandcampFilters;
 import org.schabi.newpipe.extractor.utils.Utils;
 
-import java.io.UnsupportedEncodingException;
 import java.util.List;
 
 import javax.annotation.Nonnull;

diff --git a/...abi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java b/...abi/newpipe/extractor/services/bandcamp/linkHandler/BandcampStreamLinkHandlerFactory.java
@@ -5,6 +5,7 @@
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
 import org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL;
 
@@ -49,7 +50,7 @@ public String getUrl(final String input)
         if (input.matches("\\d+")) {
             return BASE_URL + "/?show=" + input;
         } else {
-            return input;
+            return Utils.replaceHttpWithHttps(input);
         }
     }
 
@@ -71,6 +72,6 @@ public boolean onAcceptUrl(final String url) throws ParsingException {
         }
 
         // Test whether domain is supported
-        return BandcampExtractorHelper.isSupportedDomain(url);
+        return BandcampExtractorHelper.isArtistDomain(url);
     }
 }
diff --git a/...i/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java b/...i/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java
@@ -6,7 +6,6 @@
 import org.schabi.newpipe.extractor.services.media_ccc.search.filter.MediaCCCFilters;
 import org.schabi.newpipe.extractor.utils.Utils;
 
-import java.io.UnsupportedEncodingException;
 import java.util.List;
 
 import javax.annotation.Nonnull;
@@ -30,10 +29,6 @@ public String getUrl(final String query,
                          @Nonnull final List<FilterItem> contentFilter,
                          @Nullable final List<FilterItem> sortFilter)
             throws ParsingException, UnsupportedOperationException {
-        try {
-            return "https://media.ccc.de/public/events/search?q=" + Utils.encodeUrlUtf8(query);
-        } catch (final UnsupportedEncodingException e) {
-            throw new ParsingException("Could not create search string with query: " + query, e);
-        }
+        return "https://media.ccc.de/public/events/search?q=" + Utils.encodeUrlUtf8(query);
     }
 }
diff --git a/...va/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/...va/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java
@@ -38,7 +38,6 @@
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
-import java.io.UnsupportedEncodingException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -379,8 +378,7 @@ public List<Frameset> getFrames() throws ExtractionException {
     }
 
     @Nonnull
-    private String getRelatedItemsUrl(@Nonnull final List<String> tags)
-            throws UnsupportedEncodingException {
+    private String getRelatedItemsUrl(@Nonnull final List<String> tags) {
         final String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT_VIDEOS;
         final StringBuilder params = new StringBuilder();
         params.append("start=0&count=8&sort=-createdAt");

diff --git a/...bi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java b/...bi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java
@@ -8,7 +8,6 @@
 import org.schabi.newpipe.extractor.services.peertube.PeertubeHelpers;
 import org.schabi.newpipe.extractor.services.peertube.search.filter.PeertubeFilters;
 
-import java.io.UnsupportedEncodingException;
 import java.util.List;
 import java.util.Optional;
 
@@ -56,18 +55,17 @@ public String getUrl(final String searchString,
                          final List<FilterItem> selectedSortFilter,
                          final String baseUrl)
             throws ParsingException, UnsupportedOperationException {
-        try {
-            searchFilters.setSelectedSortFilter(selectedSortFilter);
-            searchFilters.setSelectedContentFilter(selectedContentFilter);
-
-            final String filterQuery = searchFilters.evaluateSelectedFilters(null);
-
-            final String endpoint = getContentFilterDependingEndpoint(selectedContentFilter);
-            return baseUrl + endpoint + "?search=" + Utils.encodeUrlUtf8(searchString)
-                    + filterQuery;
-        } catch (final UnsupportedEncodingException e) {
-            throw new ParsingException("Could not encode query", e);
+        final String endpoint;
+        if (contentFilters.isEmpty()
+                || contentFilters.get(0).equals(VIDEOS)
+                || contentFilters.get(0).equals(SEPIA_VIDEOS)) {
+            endpoint = SEARCH_ENDPOINT_VIDEOS;
+        } else if (contentFilters.get(0).equals(CHANNELS)) {
+            endpoint = SEARCH_ENDPOINT_CHANNELS;
+        } else {
+            endpoint = SEARCH_ENDPOINT_PLAYLISTS;
         }
+        return baseUrl + endpoint + "?search=" + Utils.encodeUrlUtf8(searchString);
     }
 
     private boolean isSepiaContentFilterPresent(

diff --git a/...c/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/...c/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
@@ -45,6 +45,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
+import java.util.regex.Pattern;
 
 public final class SoundcloudParsingHelper {
     // CHECKSTYLE:OFF
@@ -88,6 +89,10 @@ public final class SoundcloudParsingHelper {
     private static String clientId;
     public static final String SOUNDCLOUD_API_V2_URL = "https://api-v2.soundcloud.com/";
 
+    private static final Pattern ON_URL_PATTERN = Pattern.compile(
+        "^https?://on.soundcloud.com/[0-9a-zA-Z]+$"
+    );
+
     private SoundcloudParsingHelper() {
     }
 
@@ -185,8 +190,21 @@ public static String resolveUrlWithEmbedPlayer(final String apiUrl) throws IOExc
      */
     public static String resolveIdWithWidgetApi(final String urlString) throws IOException,
             ParsingException {
-        // Remove the tailing slash from URLs due to issues with the SoundCloud API
         String fixedUrl = urlString;
+
+        // if URL is an on.soundcloud link, do a request to resolve the redirect
+
+        if (ON_URL_PATTERN.matcher(fixedUrl).find()) {
+            try {
+                fixedUrl = NewPipe.getDownloader().head(fixedUrl).latestUrl();
+                // remove tracking params which are in the query string
+                fixedUrl = fixedUrl.split("\\?")[0];
+            } catch (final ExtractionException e) {
+                throw new ParsingException("Could not follow on.soundcloud.com redirect", e);
+            }
+        }
+
+        // Remove the tailing slash from URLs due to issues with the SoundCloud API
         if (fixedUrl.charAt(fixedUrl.length() - 1) == '/') {
             fixedUrl = fixedUrl.substring(0, fixedUrl.length() - 1);
         }

diff --git a/...rg/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java b/...rg/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java
@@ -23,7 +23,6 @@
 import org.schabi.newpipe.extractor.utils.Parser;
 
 import java.io.IOException;
-import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.Collections;
@@ -162,7 +161,7 @@ private Page getNextPageFromCurrentUrl(final String currentUrl,
     private int getOffsetFromUrl(final String url) throws ParsingException {
         try {
             return Integer.parseInt(Parser.compatParseMap(new URL(url).getQuery()).get("offset"));
-        } catch (MalformedURLException | UnsupportedEncodingException e) {
+        } catch (final MalformedURLException e) {
             throw new ParsingException("Could not get offset from page URL", e);
         }
     }