Merge branch 'mikf:master' into gwm

mikf · Jan 30, 2025 · b5561c0 · b5561c0
2 parents 7206cb2 + ba9e479
commit b5561c0
Show file tree

Hide file tree

Showing 99 changed files with 1,134 additions and 602 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,59 @@
 # Changelog
 
+## 1.28.5 - 2025-01-28
+### Extractors
+#### Additions
+- [nekohouse] add support ([#5241](https://github.com/mikf/gallery-dl/issues/5241), [#6738](https://github.com/mikf/gallery-dl/issues/6738))
+- [turboimagehost] add support for galleries ([#6855](https://github.com/mikf/gallery-dl/issues/6855))
+- [xfolio] add support ([#5514](https://github.com/mikf/gallery-dl/issues/5514), [#6351](https://github.com/mikf/gallery-dl/issues/6351), [#6837](https://github.com/mikf/gallery-dl/issues/6837))
+#### Fixes
+- [4archive] fix `TypeError`
+- [adultempire] bypass age confirmation check
+- [architizer] fix extraction
+- [artstation] avoid Cloudflare challenges ([#5817](https://github.com/mikf/gallery-dl/issues/5817), [#5658](https://github.com/mikf/gallery-dl/issues/5658), [#5564](https://github.com/mikf/gallery-dl/issues/5564), [#5554](https://github.com/mikf/gallery-dl/issues/5554))
+- [deviantart] prevent crash when accessing `premium_folder` data ([#6873](https://github.com/mikf/gallery-dl/issues/6873))
+- [fapachi] fix extraction ([#6881](https://github.com/mikf/gallery-dl/issues/6881))
+- [issuu] fix `user` extractor
+- [kemonoparty] fix `username` metadata and filtering by `tag` for `/posts` URLs ([#6833](https://github.com/mikf/gallery-dl/issues/6833))
+- [mangafox] fix chapter extraction
+- [mangahere] fix chapter extraction
+- [pixiv] fix `sanity_level` workaround ([#4327](https://github.com/mikf/gallery-dl/issues/4327))
+- [pornpics] fix pagination results from HTML pages
+- [twitter] handle exceptions during file extraction ([#6647](https://github.com/mikf/gallery-dl/issues/6647))
+- [vsco] fix `JSONDecodeError` ([#6887](https://github.com/mikf/gallery-dl/issues/6887), [#6891](https://github.com/mikf/gallery-dl/issues/6891))
+- [weebcentral] fix extraction ([#6860](https://github.com/mikf/gallery-dl/issues/6860))
+- [xhamster] fix `gallery` extractor ([#6818](https://github.com/mikf/gallery-dl/issues/6818), [#6876](https://github.com/mikf/gallery-dl/issues/6876))
+#### Improvements
+- [batoto] use `chapter_id` in default archive IDs ([#6835](https://github.com/mikf/gallery-dl/issues/6835))
+- [e621] support `e621.cc` and `e621.anthro.fr` frontend URLs ([#6809](https://github.com/mikf/gallery-dl/issues/6809))
+- [e621] prevent premature pagination end ([#6886](https://github.com/mikf/gallery-dl/issues/6886))
+- [facebook] allow accessing all metadata in `directory` format strings ([#6874](https://github.com/mikf/gallery-dl/issues/6874))
+- [hiperdex] update domain to `hiperdex.com`
+- [kemonoparty] enable filtering creator posts by tag ([#6833](https://github.com/mikf/gallery-dl/issues/6833))
+- [khinsider] add `covers` option ([#6844](https://github.com/mikf/gallery-dl/issues/6844))
+- [komikcast] update domain to `komikcast.la`
+- [lofter] improve error handling ([#6865](https://github.com/mikf/gallery-dl/issues/6865))
+- [pornpics] avoid redirect when retrieving a gallery page
+- [urlgalleries] support new URL format
+#### Metadata
+- [bunkr] extract better `filename` metadata ([#6824](https://github.com/mikf/gallery-dl/issues/6824))
+- [hiperdex] fix `description` metadata
+- [khinsider] extract more `album` metadata ([#6844](https://github.com/mikf/gallery-dl/issues/6844))
+- [mangaread] fix manga metadata extraction
+- [rule34xyz] fix `date` and `tags` metadata
+- [saint] fix metadata of `/d/` URLs
+- [toyhouse] fix `date`, `artists`, and `characters` metadata
+- [webtoons] fix `username` and `author_name` metadata
+#### Removals
+- [cohost] remove module
+- [fanleaks] remove module
+- [shimmie2] remove `tentaclerape.net`
+- [szurubooru] remove `booru.foalcon.com`
+### Miscellaneous
+- [docs] add `nix` docs to README ([#6606](https://github.com/mikf/gallery-dl/issues/6606))
+- [path] fix exception when using `--rename-to` + `--no-download` ([#6861](https://github.com/mikf/gallery-dl/issues/6861))
+- [release] include `scripts/run_tests.py` in release tarball ([#6856](https://github.com/mikf/gallery-dl/issues/6856))
+
 ## 1.28.4 - 2025-01-12
 ### Extractors
 #### Additions

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,2 +1,2 @@
-include README.rst CHANGELOG.md LICENSE
+include README.rst CHANGELOG.md LICENSE scripts/run_tests.py
 recursive-include docs *.conf
diff --git a/README.rst b/README.rst
@@ -76,9 +76,9 @@ Standalone Executable
 Prebuilt executable files with a Python interpreter and
 required Python packages included are available for
 
-- `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.28.4/gallery-dl.exe>`__
+- `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.28.5/gallery-dl.exe>`__
   (Requires `Microsoft Visual C++ Redistributable Package (x86) <https://aka.ms/vs/17/release/vc_redist.x86.exe>`__)
-- `Linux   <https://github.com/mikf/gallery-dl/releases/download/v1.28.4/gallery-dl.bin>`__
+- `Linux   <https://github.com/mikf/gallery-dl/releases/download/v1.28.5/gallery-dl.bin>`__
 
 
 Nightly Builds
@@ -117,7 +117,6 @@ Scoop
 
     scoop install gallery-dl
 
-
 Homebrew
 --------
 
@@ -174,6 +173,48 @@ This will remove the container after every use so you will always have a fresh e
 
 You can also add an alias to your shell for "gallery-dl" or create a simple bash script and drop it somewhere in your $PATH to act as a shim for this command.
 
+Nix and Home Manager
+--------------------------
+
+Adding *gallery-dl* to your system environment:
+
+.. code:: nix
+
+    environment.systemPackages = with pkgs; [
+      gallery-dl
+    ];
+
+Using :code:`nix-shell`
+
+.. code:: bash
+
+    nix-shell -p gallery-dl
+
+.. code:: bash
+
+    nix-shell -p gallery-dl --run "gallery-dl <args>"
+
+For Home Manager users, you can manage *gallery-dl* declaratively:
+
+.. code:: nix
+
+    programs.gallery-dl = {
+      enable = true;
+      settings = {
+        extractor.base-directory = "~/Downloads";
+      };
+    };
+
+Alternatively, you can just add it to :code:`home.packages` if you don't want to manage it declaratively:
+
+.. code:: nix
+
+    home.packages = with pkgs; [
+      gallery-dl
+    ];
+
+After making these changes, simply rebuild your configuration and open a new shell to have *gallery-dl* available.
+
 Usage
 =====
 

diff --git a/docs/configuration.rst b/docs/configuration.rst
@@ -401,6 +401,7 @@ Default
         ``urlgalleries``,
         ``vk``,
         ``weebcentral``,
+        ``xfolio``,
         ``zerochan``
     * ``"1.0-2.0"``
         ``flickr``,
@@ -1513,7 +1514,7 @@ Description
 
     * ``facets``: ``hashtags``, ``mentions``, and ``uris``
     * ``user``: detailed ``user`` metadata for the user referenced in the input URL
-      (See `app.bsky.actor.getProfile <https://www.docs.bsky.app/docs/api/app-bsky-actor-get-profile>`__).
+      (See `app.bsky.actor.getProfile <https://docs.bsky.app/docs/api/app-bsky-actor-get-profile>`__).
 
 
 
@@ -1526,7 +1527,7 @@ Default
 Description
     Sets the maximum depth of returned reply posts.
 
-    (See `depth` parameter of `app.bsky.feed.getPostThread <https://www.docs.bsky.app/docs/api/app-bsky-feed-get-post-thread>`__)
+    (See `depth` parameter of `app.bsky.feed.getPostThread <https://docs.bsky.app/docs/api/app-bsky-feed-get-post-thread>`__)
 
 
 extractor.bluesky.quoted
@@ -2971,6 +2972,16 @@ Description
     * ``"reverse"``: Same as ``"asc"``
 
 
+extractor.khinsider.covers
+--------------------------
+Type
+    ``bool``
+Default
+    ``false``
+Description
+    Download album cover images.
+
+
 extractor.khinsider.format
 --------------------------
 Type

diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf
@@ -366,6 +366,7 @@
         },
         "khinsider":
         {
+            "covers": false,
             "format": "mp3"
         },
         "koharu":
@@ -701,6 +702,10 @@
         {
             "sleep-request": "0.5-1.5"
         },
+        "xfolio":
+        {
+            "sleep-request": "0.5-1.5"
+        },
         "weibo":
         {
             "sleep-request": "1.0-2.0",
@@ -922,7 +927,7 @@
             "config-file"    : null,
             "enabled"        : true,
             "format"         : null,
-            "forward-cookies": false,
+            "forward-cookies": true,
             "logging"        : true,
             "module"         : null,
             "outtmpl"        : null,

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
@@ -247,12 +247,6 @@ Consider all listed sites to potentially be NSFW.
     <td>Photos, Profiles, Sets, Videos</td>
     <td><a href="https://github.com/mikf/gallery-dl#cookies">Cookies</a></td>
 </tr>
-<tr>
-    <td>Fanleaks</td>
-    <td>https://fanleaks.club/</td>
-    <td>Models, Posts</td>
-    <td></td>
-</tr>
 <tr>
     <td>Fantia</td>
     <td>https://fantia.jp/</td>
@@ -362,8 +356,8 @@ Consider all listed sites to potentially be NSFW.
     <td></td>
 </tr>
 <tr>
-    <td>Hipertoon</td>
-    <td>https://hipertoon.com/</td>
+    <td>HiperDEX</td>
+    <td>https://hiperdex.com/</td>
     <td>Artists, Chapters, Manga</td>
     <td></td>
 </tr>
@@ -507,7 +501,7 @@ Consider all listed sites to potentially be NSFW.
 </tr>
 <tr>
     <td>Komikcast</td>
-    <td>https://komikcast.cz/</td>
+    <td>https://komikcast.la/</td>
     <td>Chapters, Manga</td>
     <td></td>
 </tr>
@@ -631,6 +625,12 @@ Consider all listed sites to potentially be NSFW.
     <td>Comics, Episodes</td>
     <td></td>
 </tr>
+<tr>
+    <td>Nekohouse</td>
+    <td>https://nekohouse.su/</td>
+    <td>Posts, User Profiles</td>
+    <td></td>
+</tr>
 <tr>
     <td>Newgrounds</td>
     <td>https://www.newgrounds.com/</td>
@@ -964,7 +964,7 @@ Consider all listed sites to potentially be NSFW.
 <tr>
     <td>Turboimagehost</td>
     <td>https://www.turboimagehost.com/</td>
-    <td>individual Images</td>
+    <td>Galleries, individual Images</td>
     <td></td>
 </tr>
 <tr>
@@ -1087,6 +1087,12 @@ Consider all listed sites to potentially be NSFW.
     <td>Galleries</td>
     <td></td>
 </tr>
+<tr>
+    <td>Xfolio</td>
+    <td>https://xfolio.jp/</td>
+    <td>Series, User Profiles, Works</td>
+    <td></td>
+</tr>
 <tr>
     <td>xHamster</td>
     <td>https://xhamster.com/</td>
@@ -1451,12 +1457,6 @@ Consider all listed sites to potentially be NSFW.
     <td>Posts, Tag Searches</td>
     <td></td>
 </tr>
-<tr>
-    <td>Tentaclerape</td>
-    <td>https://tentaclerape.net/</td>
-    <td>Posts, Tag Searches</td>
-    <td></td>
-</tr>
 <tr>
     <td>Cavemanon</td>
     <td>https://booru.cavemanon.xyz/</td>
@@ -1479,12 +1479,6 @@ Consider all listed sites to potentially be NSFW.
 <tr>
     <td colspan="4"><strong>szurubooru Instances</strong></td>
 </tr>
-<tr>
-    <td>Foalcon</td>
-    <td>https://booru.foalcon.com/</td>
-    <td>Posts, Tag Searches</td>
-    <td></td>
-</tr>
 <tr>
     <td>Bcbnsfw</td>
     <td>https://booru.bcbnsfw.space/</td>

diff --git a/gallery_dl/extractor/4archive.py b/gallery_dl/extractor/4archive.py
@@ -64,7 +64,7 @@ def parse(post):
         data = {
             "name": extr('class="name">', "</span>"),
             "date": text.parse_datetime(
-                extr('class="dateTime postNum">', "<").strip(),
+                extr('class="dateTime postNum" >', "<").strip(),
                 "%Y-%m-%d %H:%M:%S"),
             "no"  : text.parse_int(extr('href="#p', '"')),
         }

diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py
@@ -51,7 +51,6 @@
     "exhentai",
     "facebook",
     "fanbox",
-    "fanleaks",
     "fantia",
     "fapello",
     "fapachi",
@@ -116,6 +115,7 @@
     "myportfolio",
     "naver",
     "naverwebtoon",
+    "nekohouse",
     "newgrounds",
     "nhentai",
     "nijie",
@@ -196,6 +196,7 @@
     "wikiart",
     "wikifeet",
     "wikimedia",
+    "xfolio",
     "xhamster",
     "xvideos",
     "yiffverse",

diff --git a/gallery_dl/extractor/adultempire.py b/gallery_dl/extractor/adultempire.py
@@ -24,6 +24,9 @@ def __init__(self, match):
         GalleryExtractor.__init__(self, match)
         self.gallery_id = match.group(2)
 
+    def _init(self):
+        self.cookies.set("ageConfirmed", "true", domain="www.adultempire.com")
+
     def metadata(self, page):
         extr = text.extract_from(page, page.index('<div id="content">'))
         return {

diff --git a/gallery_dl/extractor/architizer.py b/gallery_dl/extractor/architizer.py
@@ -32,10 +32,10 @@ def metadata(self, page):
         extr('id="Pages"', "")
 
         return {
-            "title"      : extr('data-name="', '"'),
-            "slug"       : extr('data-slug="', '"'),
-            "gid"        : extr('data-gid="', '"').rpartition(".")[2],
-            "firm"       : extr('data-firm-leaders-str="', '"'),
+            "title"      : extr("data-name='", "'"),
+            "slug"       : extr("data-slug='", "'"),
+            "gid"        : extr("data-gid='", "'").rpartition(".")[2],
+            "firm"       : extr("data-firm-leaders-str='", "'"),
             "location"   : extr("<h2>", "<").strip(),
             "type"       : text.unescape(text.remove_html(extr(
                 '<div class="title">Type</div>', '<br'))),
@@ -54,7 +54,7 @@ def images(self, page):
         return [
             (url, None)
             for url in text.extract_iter(
-                page, 'property="og:image:secure_url" content="', "?")
+                page, "property='og:image:secure_url' content='", "?")
         ]