diff --git a/src/pybuild_deps/compile_build_dependencies.py b/src/pybuild_deps/compile_build_dependencies.py index 354061f..c705f3c 100644 --- a/src/pybuild_deps/compile_build_dependencies.py +++ b/src/pybuild_deps/compile_build_dependencies.py @@ -167,7 +167,10 @@ def _find_build_dependencies( """Find build dependencies for a given ireq.""" ireq_version = get_version(ireq) for build_dep in find_build_dependencies( - ireq.name, ireq_version, raise_setuppy_parsing_exc=False + ireq.name, + ireq_version, + raise_setuppy_parsing_exc=False, + pip_session=self.repository.session, ): # The original 'find_build_dependencies' function is very naive by design. # It only returns a simple list of strings representing builds dependencies. diff --git a/src/pybuild_deps/finder.py b/src/pybuild_deps/finder.py index 1aca43b..f36f831 100644 --- a/src/pybuild_deps/finder.py +++ b/src/pybuild_deps/finder.py @@ -4,6 +4,8 @@ import tarfile +from pip._internal.network.session import PipSession + from pybuild_deps.parsers.setup_py import SetupPyParsingError from .logger import log @@ -12,7 +14,10 @@ def find_build_dependencies( - package_name, version, raise_setuppy_parsing_exc=True + package_name, + version, + raise_setuppy_parsing_exc=True, + pip_session: PipSession | None = None, ) -> list[str]: """Find build dependencies for a given package.""" file_parser_map = { @@ -21,7 +26,7 @@ def find_build_dependencies( "setup.py": parse_setup_py, } log.debug(f"retrieving source for package {package_name}=={version}") - source_path = get_package_source(package_name, version) + source_path = get_package_source(package_name, version, pip_session=pip_session) build_dependencies = [] with tarfile.open(fileobj=source_path.open("rb")) as tarball: for file_name, parser in file_parser_map.items(): diff --git a/src/pybuild_deps/parsers/requirements.py b/src/pybuild_deps/parsers/requirements.py index 10d86e8..040156a 100644 --- a/src/pybuild_deps/parsers/requirements.py +++ b/src/pybuild_deps/parsers/requirements.py @@ -14,7 +14,7 @@ ) from pybuild_deps.exceptions import PyBuildDepsError -from pybuild_deps.utils import is_pinned_requirement +from pybuild_deps.utils import is_supported_requirement def parse_requirements( @@ -31,7 +31,7 @@ def parse_requirements( filename, session, finder=finder, options=options, constraint=constraint ): ireq = install_req_from_parsed_requirement(parsed_req, isolated=isolated) - if not is_pinned_requirement(ireq): + if not is_supported_requirement(ireq): raise PyBuildDepsError( f"requirement '{ireq}' is not exact " "(pybuild-tools only supports pinned dependencies)." diff --git a/src/pybuild_deps/source.py b/src/pybuild_deps/source.py index c04afeb..7cb5cac 100644 --- a/src/pybuild_deps/source.py +++ b/src/pybuild_deps/source.py @@ -1,5 +1,7 @@ """Get source code for a given package.""" +from __future__ import annotations + import logging import tarfile from pathlib import Path @@ -7,15 +9,21 @@ from urllib.parse import urlparse import requests -from pip._internal.operations.prepare import unpack_vcs_link +from pip._internal.exceptions import InstallationError +from pip._internal.network.download import Downloader +from pip._internal.network.session import PipSession +from pip._internal.operations.prepare import unpack_url from pip._internal.req.constructors import install_req_from_req_string +from pip._internal.utils.temp_dir import global_tempdir_manager from pybuild_deps.constants import CACHE_PATH from pybuild_deps.exceptions import PyBuildDepsError -from pybuild_deps.utils import is_pinned_vcs +from pybuild_deps.utils import is_supported_requirement -def get_package_source(package_name: str, version: str) -> Path: +def get_package_source( + package_name: str, version: str, pip_session: PipSession | None = None +) -> Path: """Get source code for a given package.""" parsed_url = urlparse(version) is_url = all((parsed_url.scheme, parsed_url.netloc)) @@ -39,51 +47,51 @@ def get_package_source(package_name: str, version: str) -> Path: elif error_path.exists(): raise NotImplementedError() - if is_url: - # assume url is pointing to VCS - if it's not an error will be thrown later - return retrieve_and_save_source_from_vcs( - package_name, version, tarball_path=tarball_path, error_path=error_path - ) + url = version if is_url else get_source_url_from_pypi(package_name, version) - return retrieve_and_save_source_from_pypi( - package_name, version, tarball_path=tarball_path, error_path=error_path + return retrieve_and_save_source_from_url( + package_name, + url, + tarball_path=tarball_path, + error_path=error_path, + pip_session=pip_session, ) -def retrieve_and_save_source_from_pypi( - package_name: str, - version: str, - *, - tarball_path: Path, - error_path: Path, -): - """Retrieve package source from pypi and store it in a cache.""" - source_url = get_source_url_from_pypi(package_name, version) - response = requests.get(source_url, timeout=10) - response.raise_for_status() - tarball_path.parent.mkdir(parents=True, exist_ok=True) - tarball_path.write_bytes(response.content) - return tarball_path - - -def retrieve_and_save_source_from_vcs( +def retrieve_and_save_source_from_url( package_name: str, - version: str, + url: str, *, tarball_path: Path, error_path: Path, + pip_session: PipSession = None, ): - """Retrieve package source from VCS.""" - ireq = install_req_from_req_string(f"{package_name} @ {version}") - if not is_pinned_vcs(ireq): + """Retrieve package source from URL.""" + ireq = install_req_from_req_string(f"{package_name} @ {url}") + if not is_supported_requirement(ireq): raise PyBuildDepsError( - f"Unsupported requirement ({ireq.name} @ {ireq.link}). Url requirements " - "must use a VCS scheme like 'git+https'." + f"Unsupported requirement '{ireq.req}'. Requirement must be either pinned " + "(==), a vcs link with sha or a direct url." ) - tarball_path.parent.mkdir(parents=True, exist_ok=True) - with TemporaryDirectory() as tmp_dir, tarfile.open(tarball_path, "w") as tarball: - unpack_vcs_link(ireq.link, tmp_dir, verbosity=0) - tarball.add(tmp_dir, arcname=package_name) + + pip_session = pip_session or PipSession() + pip_downloader = Downloader(pip_session, "") + + with global_tempdir_manager(), TemporaryDirectory() as tmp_dir: + try: + unpack_url( + ireq.link, + tmp_dir, + download=pip_downloader, + verbosity=0, + ) + except InstallationError as err: + raise PyBuildDepsError( + f"Unable to unpack '{ireq.req}'. Is {ireq.link} a python package?" + ) from err + tarball_path.parent.mkdir(parents=True, exist_ok=True) + with tarfile.open(tarball_path, "w:gz") as tarball: + tarball.add(tmp_dir, arcname=package_name) return tarball_path diff --git a/src/pybuild_deps/utils.py b/src/pybuild_deps/utils.py index abb3421..81df8a2 100644 --- a/src/pybuild_deps/utils.py +++ b/src/pybuild_deps/utils.py @@ -10,22 +10,21 @@ def get_version(ireq: InstallRequirement): """Get version string from InstallRequirement.""" - if not is_pinned_requirement(ireq): - raise PyBuildDepsError( - f"requirement '{ireq}' is not exact " - "(pybuild-tools only supports pinned dependencies)." - ) - if ireq.link and ireq.link.is_vcs: + if not is_supported_requirement(ireq): + raise PyBuildDepsError(f"requirement '{ireq}' is not exact.") + if ireq.req.url: return ireq.req.url return next(iter(ireq.specifier)).version -def is_pinned_requirement(ireq: InstallRequirement): - """Returns True if requirement is pinned or vcs.""" - return _is_pinned_requirement(ireq) or is_pinned_vcs(ireq) +def is_supported_requirement(ireq: InstallRequirement): + """Returns True if requirement is pinned, vcs poiting to a SHA or a direct url.""" + return ( + _is_pinned_requirement(ireq) or _is_pinned_vcs(ireq) or _is_non_vcs_link(ireq) + ) -def is_pinned_vcs(ireq: InstallRequirement): +def _is_pinned_vcs(ireq: InstallRequirement): """Check if given ireq is a pinned vcs dependency.""" if not ireq.link: return False @@ -37,3 +36,9 @@ def is_pinned_vcs(ireq: InstallRequirement): # some_project.git@da39a3ee5e6b4b0d3255bfef95601890afd80709 # https://pip.pypa.io/en/latest/topics/vcs-support/ return parts == 2 + + +def _is_non_vcs_link(ireq: InstallRequirement): + if not ireq.link: + return False + return not ireq.link.is_vcs diff --git a/tests/test_main.py b/tests/test_main.py index 1320cb2..9d019b6 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -57,6 +57,27 @@ def test_main_succeeds(runner: CliRunner) -> None: "setuptools-rust>=0.11.4", ], ), + ( + "cryptography", + "git+https://github.com/pyca/cryptography@41.0.5", + [ + "setuptools>=61.0.0", + "wheel", + "cffi>=1.12; platform_python_implementation != 'PyPy'", + "setuptools-rust>=0.11.4", + ], + ), + ( + "cryptography", + "https://github.com/pyca/cryptography/archive/refs/tags/43.0.0.tar.gz", + [ + "maturin>=1,<2", + "cffi>=1.12; platform_python_implementation != 'PyPy'", + "setuptools", + ], + ), + ("azure-identity", "1.14.1", []), + ("debugpy", "1.8.5", ["wheel", "setuptools"]), ], ) def test_find_build_deps( @@ -68,7 +89,7 @@ def test_find_build_deps( assert result.exit_code == 0 assert result.stdout.splitlines() == expected_deps assert cache.exists() - # repeating the same test to cover a cached version + # repeating the same test to cover the cached version result = runner.invoke(main.cli, args=["find-build-deps", package_name, version]) assert result.exit_code == 0 assert result.stdout.splitlines() == expected_deps @@ -90,12 +111,12 @@ def test_find_build_deps( ( "some-package", "git+https://example.com", - "Unsupported requirement (some-package @ git+https://example.com). Url requirements must use a VCS scheme like 'git+https'.", # noqa: E501 + "Unsupported requirement 'some-package@ git+https://example.com'. Requirement must be either pinned (==), a vcs link with sha or a direct url.", # noqa: E501 ), ( "cryptography", "git+https://github.com/pyca/cryptography", - "Unsupported requirement (cryptography @ git+https://github.com/pyca/cryptography). Url requirements must use a VCS scheme like 'git+https'.", # noqa: E501 + "Unsupported requirement 'cryptography@ git+https://github.com/pyca/cryptography'. Requirement must be either pinned (==), a vcs link with sha or a direct url.", # noqa: E501 ), ], ) diff --git a/tests/test_utils.py b/tests/test_utils.py index 125fdcc..9b8ced0 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,7 +3,7 @@ import pytest from pip._internal.req.constructors import install_req_from_req_string -from pybuild_deps.utils import get_version, is_pinned_requirement +from pybuild_deps.utils import get_version, is_supported_requirement @pytest.mark.parametrize( @@ -16,7 +16,7 @@ def test_is_pinned_or_vcs(req): """Ensure pinned or vcs dependencies are properly detected.""" ireq = install_req_from_req_string(req) - assert is_pinned_requirement(ireq) + assert is_supported_requirement(ireq) @pytest.mark.parametrize( @@ -24,15 +24,13 @@ def test_is_pinned_or_vcs(req): [ "requests>1.2.3", "requests @ git+https://github.com/psf/requests", - "requests @ https://example.com", - "requests @ https://github.com/psf/requests@some-commit-sha", "requests", ], ) def test_not_pinned_or_vcs(req): """Negative test for 'is_pinned_or_vcs'.""" ireq = install_req_from_req_string(req) - assert not is_pinned_requirement(ireq) + assert not is_supported_requirement(ireq) def test_get_version_url():