feature: support constraints for more consistent results

src/pybuild_deps/compile_build_dependencies.py

@@ -17,6 +17,7 @@
 from pip._vendor.resolvelib.resolvers import ResolutionImpossible
 from piptools.repositories import PyPIRepository
 from piptools.resolver import BacktrackingResolver
+from piptools.utils import key_from_ireq
 
 from .exceptions import UnsolvableDependenciesError
 from .finder import find_build_dependencies
@@ -30,29 +31,38 @@ class BuildDependencyCompiler:
     def __init__(self, repository: PyPIRepository) -> None:
         self.repository = repository
         self.resolver = None
-        self.dependency_cache = {}
 
     def resolve(
         self,
         install_requirements: Iterable[InstallRequirement],
+        existing_constraints: dict[str, InstallRequirement] | None = None,
+        dependency_cache: dict[str, InstallRequirement] | None = None,
     ) -> set[InstallRequirement]:
         """Resolve all build dependencies for a given set of dependencies."""
         all_build_deps = []
+
+        # reuse or initialize constraints (following what piptools expects downstream)
+        # and our dependency cache
+        existing_constraints = existing_constraints or {
+            key_from_ireq(ireq): ireq for ireq in install_requirements
+        }
+        dependency_cache = dependency_cache or {}
+
         for ireq in install_requirements:
             log.info("=" * 80)
             log.info(str(ireq))
             log.info("-" * 80)
             req_version = get_version(ireq)
             ireq_name = f"{ireq.name}=={req_version}"
-            if ireq_name in self.dependency_cache:
-                all_build_deps.extend(self.dependency_cache[ireq_name])
-                log.debug(f"{ireq} exists in our cache, moving on...")
+            if ireq_name in dependency_cache:
+                all_build_deps.extend(dependency_cache[ireq_name])
+                log.debug(f"{ireq} was already solved, moving on...")
                 continue
             raw_build_dependencies = find_build_dependencies(
                 ireq.name, req_version, raise_setuppy_parsing_exc=False
             )
             if not raw_build_dependencies:
-                self.dependency_cache[ireq_name] = set()
+                dependency_cache[ireq_name] = set()
                 continue
             # 'find_build_dependencies' is very naive - by design - and only returns
             # a simple list of strings representing build (or transitive) dependencies.
@@ -63,32 +73,60 @@ def resolve(
                 partial(install_req_from_req_string, comes_from=ireq.name),
                 raw_build_dependencies,
             )
-            build_dependencies = self._resolve_with_piptools(
-                package=ireq_name, ireqs=build_ireqs
-            )
+            try:
+                # Attempt to resolve ireq's transitive dependencies using
+                # runtime requirements as constraint. This is equivalent to
+                # running "pip install -c constraints-file.txt".
+                build_dependencies = self._resolve_with_piptools(
+                    package=ireq_name,
+                    ireqs=build_ireqs,
+                    constraints=existing_constraints,
+                )
+            except UnsolvableDependenciesError:
+                # Being unsolvable on the previous step doesn't mean a transitive
+                # dependency is actually unsolvable. Per PEP-517, transitive
+                # dependencies are built in isolated environments. We only
+                # try building with constraints to avoid ending up with an unnecessarily
+                # large list of dependencies to manage.
+
+                # If this step fails, the same exception will bubble up and explode
+                # in an error.
+                build_dependencies = self._resolve_with_piptools(
+                    package=ireq_name,
+                    ireqs=build_ireqs,
+                )
+
             # dependencies of build dependencies might have their own build
             # dependencies, so let's recursively search for those.
             build_deps_qty = 0
             while len(build_dependencies) != build_deps_qty:
                 build_deps_qty = len(build_dependencies)
-                build_dependencies |= set(self.resolve(build_dependencies))
-            self.dependency_cache[ireq_name] = build_dependencies
+                build_dependencies |= self.resolve(
+                    build_dependencies,
+                    existing_constraints=existing_constraints,
+                    dependency_cache=dependency_cache,
+                )
+
+            dependency_cache[ireq_name] = build_dependencies
 
             all_build_deps.extend(build_dependencies)
 
         return deduplicate_install_requirements(all_build_deps)
 
     def _resolve_with_piptools(
-        self, package: str, ireqs: Iterable[InstallRequirement]
+        self,
+        package: str,
+        ireqs: Iterable[InstallRequirement],
+        constraints: set[InstallRequirement],
     ) -> set[InstallRequirement]:
         # backup unsafe data before overriding resolver, we will need it later
         # on piptools writer to export the file
         unsafe_packages = getattr(self.resolver, "unsafe_packages", set())
         unsafe_constraints = getattr(self.resolver, "unsafe_constraints", set())
-        # override resolver - we only want the latest and greatest
+        # override resolver - we don't want references from other
         self.resolver = BacktrackingResolver(
             constraints=ireqs,
-            existing_constraints={},
+            existing_constraints=constraints,
             repository=self.repository,
             allow_unsafe=True,
         )