feature: add support for zipped sdists and pinned url dependencies

Modified pybuild-deps internals to rely more on pip internals, which
gave us both support for url dependencies and zip archived packages
for "free". This should fix (or at least cover most of) issues #188
and #187.

We will need to watch closely when pip releases new versions in order
fix breaking changes in its internal APIs.

src/pybuild_deps/compile_build_dependencies.py

@@ -167,7 +167,10 @@ def _find_build_dependencies(
         """Find build dependencies for a given ireq."""
         ireq_version = get_version(ireq)
         for build_dep in find_build_dependencies(
-            ireq.name, ireq_version, raise_setuppy_parsing_exc=False
+            ireq.name,
+            ireq_version,
+            raise_setuppy_parsing_exc=False,
+            pip_session=self.repository.session,
         ):
             # The original 'find_build_dependencies' function is very naive by design.
             # It only returns a simple list of strings representing builds dependencies.

src/pybuild_deps/finder.py

@@ -4,6 +4,8 @@
 
 import tarfile
 
+from pip._internal.network.session import PipSession
+
 from pybuild_deps.parsers.setup_py import SetupPyParsingError
 
 from .logger import log
@@ -12,7 +14,10 @@
 
 
 def find_build_dependencies(
-    package_name, version, raise_setuppy_parsing_exc=True
+    package_name,
+    version,
+    raise_setuppy_parsing_exc=True,
+    pip_session: PipSession | None = None,
 ) -> list[str]:
     """Find build dependencies for a given package."""
     file_parser_map = {
@@ -21,7 +26,7 @@ def find_build_dependencies(
         "setup.py": parse_setup_py,
     }
     log.debug(f"retrieving source for package {package_name}=={version}")
-    source_path = get_package_source(package_name, version)
+    source_path = get_package_source(package_name, version, pip_session=pip_session)
     build_dependencies = []
     with tarfile.open(fileobj=source_path.open("rb")) as tarball:
         for file_name, parser in file_parser_map.items():

src/pybuild_deps/parsers/requirements.py

@@ -14,7 +14,7 @@
 )
 
 from pybuild_deps.exceptions import PyBuildDepsError
-from pybuild_deps.utils import is_pinned_requirement
+from pybuild_deps.utils import is_supported_requirement
 
 
 def parse_requirements(
@@ -31,7 +31,7 @@ def parse_requirements(
         filename, session, finder=finder, options=options, constraint=constraint
     ):
         ireq = install_req_from_parsed_requirement(parsed_req, isolated=isolated)
-        if not is_pinned_requirement(ireq):
+        if not is_supported_requirement(ireq):
             raise PyBuildDepsError(
                 f"requirement '{ireq}' is not exact "
                 "(pybuild-tools only supports pinned dependencies)."

src/pybuild_deps/source.py

@@ -1,21 +1,29 @@
 """Get source code for a given package."""
 
+from __future__ import annotations
+
 import logging
 import tarfile
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from urllib.parse import urlparse
 
 import requests
-from pip._internal.operations.prepare import unpack_vcs_link
+from pip._internal.exceptions import InstallationError
+from pip._internal.network.download import Downloader
+from pip._internal.network.session import PipSession
+from pip._internal.operations.prepare import unpack_url
 from pip._internal.req.constructors import install_req_from_req_string
+from pip._internal.utils.temp_dir import global_tempdir_manager
 
 from pybuild_deps.constants import CACHE_PATH
 from pybuild_deps.exceptions import PyBuildDepsError
-from pybuild_deps.utils import is_pinned_vcs
+from pybuild_deps.utils import is_supported_requirement
 
 
-def get_package_source(package_name: str, version: str) -> Path:
+def get_package_source(
+    package_name: str, version: str, pip_session: PipSession | None = None
+) -> Path:
     """Get source code for a given package."""
     parsed_url = urlparse(version)
     is_url = all((parsed_url.scheme, parsed_url.netloc))
@@ -39,51 +47,51 @@ def get_package_source(package_name: str, version: str) -> Path:
     elif error_path.exists():
         raise NotImplementedError()
 
-    if is_url:
-        # assume url is pointing to VCS - if it's not an error will be thrown later
-        return retrieve_and_save_source_from_vcs(
-            package_name, version, tarball_path=tarball_path, error_path=error_path
-        )
+    url = version if is_url else get_source_url_from_pypi(package_name, version)
 
-    return retrieve_and_save_source_from_pypi(
-        package_name, version, tarball_path=tarball_path, error_path=error_path
+    return retrieve_and_save_source_from_url(
+        package_name,
+        url,
+        tarball_path=tarball_path,
+        error_path=error_path,
+        pip_session=pip_session,
     )
 
 
-def retrieve_and_save_source_from_pypi(
-    package_name: str,
-    version: str,
-    *,
-    tarball_path: Path,
-    error_path: Path,
-):
-    """Retrieve package source from pypi and store it in a cache."""
-    source_url = get_source_url_from_pypi(package_name, version)
-    response = requests.get(source_url, timeout=10)
-    response.raise_for_status()
-    tarball_path.parent.mkdir(parents=True, exist_ok=True)
-    tarball_path.write_bytes(response.content)
-    return tarball_path
-
-
-def retrieve_and_save_source_from_vcs(
+def retrieve_and_save_source_from_url(
     package_name: str,
-    version: str,
+    url: str,
     *,
     tarball_path: Path,
     error_path: Path,
+    pip_session: PipSession = None,
 ):
-    """Retrieve package source from VCS."""
-    ireq = install_req_from_req_string(f"{package_name} @ {version}")
-    if not is_pinned_vcs(ireq):
+    """Retrieve package source from URL."""
+    ireq = install_req_from_req_string(f"{package_name} @ {url}")
+    if not is_supported_requirement(ireq):
         raise PyBuildDepsError(
-            f"Unsupported requirement ({ireq.name} @ {ireq.link}). Url requirements "
-            "must use a VCS scheme like 'git+https'."
+            f"Unsupported requirement '{ireq.req}'. Requirement must be either pinned "
+            "(==), a vcs link with sha or a direct url."
         )
-    tarball_path.parent.mkdir(parents=True, exist_ok=True)
-    with TemporaryDirectory() as tmp_dir, tarfile.open(tarball_path, "w") as tarball:
-        unpack_vcs_link(ireq.link, tmp_dir, verbosity=0)
-        tarball.add(tmp_dir, arcname=package_name)
+
+    pip_session = pip_session or PipSession()
+    pip_downloader = Downloader(pip_session, "")
+
+    with global_tempdir_manager(), TemporaryDirectory() as tmp_dir:
+        try:
+            unpack_url(
+                ireq.link,
+                tmp_dir,
+                download=pip_downloader,
+                verbosity=0,
+            )
+        except InstallationError as err:
+            raise PyBuildDepsError(
+                f"Unable to unpack '{ireq.req}'. Is {ireq.link} a python package?"
+            ) from err
+        tarball_path.parent.mkdir(parents=True, exist_ok=True)
+        with tarfile.open(tarball_path, "w:gz") as tarball:
+            tarball.add(tmp_dir, arcname=package_name)
     return tarball_path
 
 

src/pybuild_deps/utils.py

@@ -10,22 +10,21 @@
 
 def get_version(ireq: InstallRequirement):
     """Get version string from InstallRequirement."""
-    if not is_pinned_requirement(ireq):
-        raise PyBuildDepsError(
-            f"requirement '{ireq}' is not exact "
-            "(pybuild-tools only supports pinned dependencies)."
-        )
-    if ireq.link and ireq.link.is_vcs:
+    if not is_supported_requirement(ireq):
+        raise PyBuildDepsError(f"requirement '{ireq}' is not exact.")
+    if ireq.req.url:
         return ireq.req.url
     return next(iter(ireq.specifier)).version
 
 
-def is_pinned_requirement(ireq: InstallRequirement):
-    """Returns True if requirement is pinned or vcs."""
-    return _is_pinned_requirement(ireq) or is_pinned_vcs(ireq)
+def is_supported_requirement(ireq: InstallRequirement):
+    """Returns True if requirement is pinned, vcs poiting to a SHA or a direct url."""
+    return (
+        _is_pinned_requirement(ireq) or _is_pinned_vcs(ireq) or _is_non_vcs_link(ireq)
+    )
 
 
-def is_pinned_vcs(ireq: InstallRequirement):
+def _is_pinned_vcs(ireq: InstallRequirement):
     """Check if given ireq is a pinned vcs dependency."""
     if not ireq.link:
         return False
@@ -37,3 +36,9 @@ def is_pinned_vcs(ireq: InstallRequirement):
     # some_project.git@da39a3ee5e6b4b0d3255bfef95601890afd80709
     # https://pip.pypa.io/en/latest/topics/vcs-support/
     return parts == 2
+
+
+def _is_non_vcs_link(ireq: InstallRequirement):
+    if not ireq.link:
+        return False
+    return not ireq.link.is_vcs

tests/test_main.py

@@ -57,6 +57,27 @@ def test_main_succeeds(runner: CliRunner) -> None:
                 "setuptools-rust>=0.11.4",
             ],
         ),
+        (
+            "cryptography",
+            "git+https://github.com/pyca/[email protected]",
+            [
+                "setuptools>=61.0.0",
+                "wheel",
+                "cffi>=1.12; platform_python_implementation != 'PyPy'",
+                "setuptools-rust>=0.11.4",
+            ],
+        ),
+        (
+            "cryptography",
+            "https://github.com/pyca/cryptography/archive/refs/tags/43.0.0.tar.gz",
+            [
+                "maturin>=1,<2",
+                "cffi>=1.12; platform_python_implementation != 'PyPy'",
+                "setuptools",
+            ],
+        ),
+        ("azure-identity", "1.14.1", []),
+        ("debugpy", "1.8.5", ["wheel", "setuptools"]),
     ],
 )
 def test_find_build_deps(
@@ -68,7 +89,7 @@ def test_find_build_deps(
     assert result.exit_code == 0
     assert result.stdout.splitlines() == expected_deps
     assert cache.exists()
-    # repeating the same test to cover a cached version
+    # repeating the same test to cover the cached version
     result = runner.invoke(main.cli, args=["find-build-deps", package_name, version])
     assert result.exit_code == 0
     assert result.stdout.splitlines() == expected_deps
@@ -90,12 +111,12 @@ def test_find_build_deps(
         (
             "some-package",
             "git+https://example.com",
-            "Unsupported requirement (some-package @ git+https://example.com). Url requirements must use a VCS scheme like 'git+https'.",  # noqa: E501
+            "Unsupported requirement 'some-package@ git+https://example.com'. Requirement must be either pinned (==), a vcs link with sha or a direct url.",  # noqa: E501
         ),
         (
             "cryptography",
             "git+https://github.com/pyca/cryptography",
-            "Unsupported requirement (cryptography @ git+https://github.com/pyca/cryptography). Url requirements must use a VCS scheme like 'git+https'.",  # noqa: E501
+            "Unsupported requirement 'cryptography@ git+https://github.com/pyca/cryptography'. Requirement must be either pinned (==), a vcs link with sha or a direct url.",  # noqa: E501
         ),
     ],
 )

tests/test_utils.py

@@ -3,7 +3,7 @@
 import pytest
 from pip._internal.req.constructors import install_req_from_req_string
 
-from pybuild_deps.utils import get_version, is_pinned_requirement
+from pybuild_deps.utils import get_version, is_supported_requirement
 
 
 @pytest.mark.parametrize(
@@ -16,23 +16,21 @@
 def test_is_pinned_or_vcs(req):
     """Ensure pinned or vcs dependencies are properly detected."""
     ireq = install_req_from_req_string(req)
-    assert is_pinned_requirement(ireq)
+    assert is_supported_requirement(ireq)
 
 
 @pytest.mark.parametrize(
     "req",
     [
         "requests>1.2.3",
         "requests @ git+https://github.com/psf/requests",
-        "requests @ https://example.com",
-        "requests @ https://github.com/psf/requests@some-commit-sha",
         "requests",
     ],
 )
 def test_not_pinned_or_vcs(req):
     """Negative test for 'is_pinned_or_vcs'."""
     ireq = install_req_from_req_string(req)
-    assert not is_pinned_requirement(ireq)
+    assert not is_supported_requirement(ireq)
 
 
 def test_get_version_url():