feat: Add --test-mode for resilient bootstrap with failure handling

Add --test-mode flag that enables resilient bootstrapping by marking failed
packages as pre-built and continuing until all packages are processed. Uses
optimal n+1 retry logic with comprehensive failure reporting including exception
types, messages, and per-package context.

Benefits:
- Discover all build failures in one run rather than stopping on first failure
- Support mixed source/binary dependency workflows
- Better error context for debugging failed builds
- Cleaner API boundaries between configuration and runtime context

Fixes #713

Co-developed-with: Cursor IDE with Claude 4.0 Sonnet

Signed-off-by: Lalatendu Mohanty <[email protected]>

Author: LalatenduMohanty

src/fromager/bootstrapper.py

@@ -97,6 +97,13 @@ def _get_requirements_from_args(
     default=False,
     help="Skip generating constraints.txt file to allow building collections with conflicting versions",
 )
+@click.option(
+    "--test-mode",
+    "test_mode",
+    is_flag=True,
+    default=False,
+    help="Test mode: mark failed packages as pre-built and continue, report failures at end",
+)
 @click.argument("toplevel", nargs=-1)
 @click.pass_obj
 def bootstrap(
@@ -106,6 +113,7 @@ def bootstrap(
     cache_wheel_server_url: str | None,
     sdist_only: bool,
     skip_constraints: bool,
+    test_mode: bool,
     toplevel: list[str],
 ) -> None:
     """Compute and build the dependencies of a set of requirements recursively
@@ -116,6 +124,11 @@ def bootstrap(
     """
     logger.info(f"cache wheel server url: {cache_wheel_server_url}")
 
+    if test_mode:
+        logger.info(
+            "test mode enabled: will mark failed packages as pre-built and continue"
+        )
+
     to_build = _get_requirements_from_args(toplevel, requirements_files)
     if not to_build:
         raise RuntimeError(
@@ -148,6 +161,7 @@ def bootstrap(
             prev_graph,
             cache_wheel_server_url,
             sdist_only=sdist_only,
+            test_mode=test_mode,
         )
 
         # we need to resolve all the top level dependencies before we start bootstrapping.
@@ -183,9 +197,29 @@ def bootstrap(
 
         for req in to_build:
             token = requirement_ctxvar.set(req)
-            bt.bootstrap(req, requirements_file.RequirementType.TOP_LEVEL)
-            progressbar.update()
-            requirement_ctxvar.reset(token)
+            try:
+                bt.bootstrap(req, requirements_file.RequirementType.TOP_LEVEL)
+                progressbar.update()
+                if test_mode:
+                    logger.info("Successfully processed: %s", req)
+            except Exception as err:
+                if test_mode:
+                    # Test mode: record error, log, and continue processing
+                    logger.error(
+                        "test mode: failed to process %s: %s",
+                        req,
+                        err,
+                        exc_info=True,  # Full traceback to debug log
+                    )
+                    bt.failed_builds.append(
+                        bootstrapper.BuildResult.failure(req=req, exception=err)
+                    )
+                    progressbar.update()  # Update progress even on failure
+                else:
+                    # Normal mode: re-raise the exception (fail-fast)
+                    raise
+            finally:
+                requirement_ctxvar.reset(token)
 
     constraints_filename = wkctx.work_dir / "constraints.txt"
     if skip_constraints:
@@ -200,7 +234,57 @@ def bootstrap(
 
     logger.debug("match_py_req LRU cache: %r", resolver.match_py_req.cache_info())
 
-    metrics.summarize(wkctx, "Bootstrapping")
+    # Test mode summary reporting
+    if test_mode:
+        if bt.failed_builds:
+            # Use repository's logging pattern for error reporting
+            logger.error("test mode: the following packages failed to build:")
+            for failure in sorted(
+                bt.failed_builds, key=lambda f: str(f.req) if f.req else ""
+            ):
+                if failure.req and failure.resolved_version:
+                    logger.error(
+                        "  - %s==%s",
+                        failure.req,
+                        failure.resolved_version,
+                    )
+                    if failure.exception_type:
+                        logger.error(
+                            "    Error: %s: %s",
+                            failure.exception_type,
+                            failure.exception_message,
+                        )
+                else:
+                    logger.error("  - unknown package (missing context)")
+
+            # Categorize failures by exception type for better analysis
+            failure_types: dict[str, list[str]] = {}
+            for failure in bt.failed_builds:
+                exc_type = failure.exception_type or "Unknown"
+                pkg_name = (
+                    f"{failure.req}=={failure.resolved_version}"
+                    if failure.req and failure.resolved_version
+                    else "unknown"
+                )
+                failure_types.setdefault(exc_type, []).append(pkg_name)
+
+            logger.error("")
+            logger.error("test mode: failure breakdown by type:")
+            for exc_type, packages in sorted(failure_types.items()):
+                logger.error("  %s: %d package(s)", exc_type, len(packages))
+
+            logger.error(
+                "test mode: %d package(s) failed to build", len(bt.failed_builds)
+            )
+            # Follow repository's error exit pattern like __main__.py and lint.py
+            raise SystemExit(
+                f"Test mode completed with {len(bt.failed_builds)} build failures"
+            )
+        else:
+            logger.info("test mode: all packages built successfully")
+        metrics.summarize(wkctx, "Test Mode Bootstrapping")
+    else:
+        metrics.summarize(wkctx, "Bootstrapping")
 
 
 def write_constraints_file(
@@ -480,6 +564,9 @@ def bootstrap_parallel(
     remaining wheels in parallel. The bootstrap step downloads sdists
     and builds build-time dependency in serial. The build-parallel step
     builds the remaining wheels in parallel.
+
+    Note: --test-mode is not supported with bootstrap-parallel. Use
+    'bootstrap --test-mode' for comprehensive failure testing.
     """
     # Do not remove build environments in bootstrap phase to speed up the
     # parallel build phase.
@@ -495,6 +582,7 @@ def bootstrap_parallel(
         cache_wheel_server_url=cache_wheel_server_url,
         sdist_only=True,
         skip_constraints=skip_constraints,
+        test_mode=False,
         toplevel=toplevel,
     )
 

src/fromager/commands/bootstrap.py

@@ -125,7 +125,7 @@ def _find_customized_nodes(
     """Filter nodes to find only those with customizations."""
     customized_nodes: list[DependencyNode] = []
     for node in nodes:
-        pbi = wkctx.settings.package_build_info(node.canonicalized_name)
+        pbi = wkctx.package_build_info(node.canonicalized_name)
         if node.canonicalized_name != ROOT and pbi.has_customizations:
             customized_nodes.append(node)
     return customized_nodes
@@ -161,7 +161,7 @@ def _find_customized_dependencies_for_node(
                 continue
 
             child = edge.destination_node
-            child_pbi = wkctx.settings.package_build_info(child.canonicalized_name)
+            child_pbi = wkctx.package_build_info(child.canonicalized_name)
             new_path = path + [current_node.key]
 
             # Use the first requirement we encounter in the path
@@ -277,7 +277,7 @@ def get_node_id(node: str) -> str:
         if not name:
             node_type.append("toplevel")
         else:
-            pbi = wkctx.settings.package_build_info(name)
+            pbi = wkctx.package_build_info(name)
             all_patches: PatchMap = pbi.get_all_patches()
 
             if node.pre_built:

src/fromager/commands/graph.py

@@ -65,7 +65,7 @@ def list_overrides(
     export_data = []
 
     for name in overridden_packages:
-        pbi = wkctx.settings.package_build_info(name)
+        pbi = wkctx.package_build_info(name)
         ps = wkctx.settings.package_setting(name)
 
         plugin_hooks: list[str] = []

src/fromager/commands/list_overrides.py

@@ -164,7 +164,7 @@ def package_build_info(
             name = package.name
         else:
             name = package
-        return self.settings.package_build_info(name)
+        return self.settings.package_build_info(name, self)
 
     def setup(self) -> None:
         # The work dir must already exist, so don't try to create it.

src/fromager/context.py

@@ -624,12 +624,26 @@ def get_available_memory_gib() -> float:
 
 
 class PackageBuildInfo:
-    """Package build information
+    """Variant-aware package build configuration and metadata.
 
-    Public API for PackageSettings with i
+    Primary public API for accessing package-specific settings during the build
+    process. Combines static configuration from YAML files with runtime context
+    to provide variant-specific (cpu, cuda, etc.) build information.
+
+    Key responsibilities:
+    - Determine if package should be built or use pre-built wheels
+    - Provide patches to apply for specific versions
+    - Configure build environment (parallel jobs, environment variables)
+    - Manage package customizations (plugins, custom download URLs)
+    - Calculate build tags from changelogs for wheel versioning
+
+    Instances are cached per package and accessed via ``WorkContext.package_build_info()``.
     """
 
-    def __init__(self, settings: Settings, ps: PackageSettings) -> None:
+    def __init__(
+        self, settings: Settings, ps: PackageSettings, ctx: context.WorkContext
+    ) -> None:
+        self._ctx = ctx
         self._variant = typing.cast(Variant, settings.variant)
         self._patches_dir = settings.patches_dir
         self._variant_changelog = settings.variant_changelog()
@@ -744,7 +758,7 @@ def has_customizations(self) -> bool:
 
     @property
     def pre_built(self) -> bool:
-        """Does the variant use pre-build wheels?"""
+        """Does the variant use pre-built wheels?"""
         vi = self._ps.variants.get(self.variant)
         if vi is not None:
             return vi.pre_built
@@ -1146,23 +1160,27 @@ def package_setting(self, package: str | Package) -> PackageSettings:
             self._package_settings[package] = ps
         return ps
 
-    def package_build_info(self, package: str | Package) -> PackageBuildInfo:
+    def package_build_info(
+        self, package: str | Package, ctx: context.WorkContext
+    ) -> PackageBuildInfo:
         """Get (cached) PackageBuildInfo for package and current variant"""
         package = Package(canonicalize_name(package, validate=True))
         pbi = self._pbi_cache.get(package)
         if pbi is None:
             ps = self.package_setting(package)
-            pbi = PackageBuildInfo(self, ps)
+            pbi = PackageBuildInfo(self, ps, ctx)
             self._pbi_cache[package] = pbi
         return pbi
 
     def list_pre_built(self) -> set[Package]:
-        """List packages marked as pre-built"""
-        return set(
-            name
-            for name in self._package_settings
-            if self.package_build_info(name).pre_built
-        )
+        """List packages marked as pre-built by configuration"""
+        result = set()
+        for name in self._package_settings:
+            ps = self._package_settings[name]
+            vi = ps.variants.get(self._variant)
+            if vi is not None and vi.pre_built:
+                result.add(name)
+        return result
 
     def list_overrides(self) -> set[Package]:
         """List packages with overrides

src/fromager/packagesettings.py

@@ -0,0 +1,91 @@
+"""Tests for bootstrap --test-mode functionality.
+
+Tests for test mode failure tracking and BuildResult.
+"""
+
+from unittest import mock
+
+import pytest
+from packaging.requirements import Requirement
+from packaging.version import Version
+
+from fromager import bootstrapper
+from fromager.context import WorkContext
+
+
+class MockBuildError(Exception):
+    """Mock exception for simulating build failures."""
+
+    pass
+
+
+def test_test_mode_tracks_complete_failures(tmp_context: WorkContext) -> None:
+    """Test that test mode tracks failures with full context when both build and fallback fail."""
+    bt = bootstrapper.Bootstrapper(tmp_context, test_mode=True)
+
+    # Mock to always fail
+    def mock_build_wheel_and_sdist(req, version, pbi, build_sdist_only):
+        raise MockBuildError(f"Build failed for {req.name}")
+
+    with mock.patch.object(
+        bt, "_build_wheel_and_sdist", side_effect=mock_build_wheel_and_sdist
+    ):
+        req = Requirement("broken-package==1.0")
+        version = Version("1.0")
+        pbi = tmp_context.package_build_info(req)
+
+        result = bt._build_package(req, version, pbi, build_sdist_only=False)
+
+        # Verify complete failure is tracked with full context
+        assert result.failed
+        assert result.req == req
+        assert result.resolved_version == version
+        assert result.exception_type == "MockBuildError"
+        assert result.exception_message is not None
+        assert "Build failed for broken-package" in result.exception_message
+
+        # Verify failure is in failed_builds list
+        assert len(bt.failed_builds) == 1
+        failed_build = bt.failed_builds[0]
+        assert failed_build.req is not None
+        assert failed_build.req.name == "broken-package"
+
+
+def test_normal_mode_still_fails_fast(tmp_context: WorkContext) -> None:
+    """Test that normal mode (test_mode=False) still raises exceptions immediately."""
+    bt = bootstrapper.Bootstrapper(tmp_context, test_mode=False)
+
+    def mock_build_wheel_and_sdist(req, version, pbi, build_sdist_only):
+        raise MockBuildError(f"Build failed for {req.name}")
+
+    with mock.patch.object(
+        bt, "_build_wheel_and_sdist", side_effect=mock_build_wheel_and_sdist
+    ):
+        req = Requirement("failing-package==1.0")
+        version = Version("1.0")
+        pbi = tmp_context.package_build_info(req)
+
+        # Should raise immediately in normal mode
+        with pytest.raises(MockBuildError, match="Build failed for failing-package"):
+            bt._build_package(req, version, pbi, build_sdist_only=False)
+
+
+def test_build_result_captures_exception_context() -> None:
+    """Test that BuildResult.failure() properly captures exception context."""
+    req = Requirement("test-package>=1.0")
+    version = Version("1.2.3")
+    exception = ValueError("Something went wrong")
+
+    result = bootstrapper.BuildResult.failure(
+        req=req, resolved_version=version, exception=exception
+    )
+
+    # Verify all context is captured
+    assert result.failed
+    assert result.req == req
+    assert result.resolved_version == version
+    assert result.exception is exception
+    assert result.exception_type == "ValueError"
+    assert result.exception_message == "Something went wrong"
+    assert result.wheel_filename is None
+    assert result.sdist_filename is None

tests/test_bootstrap_test_mode.py

@@ -17,5 +17,7 @@ def test_bootstrap_parallel_options() -> None:
     # graph_file internally.
     expected.discard("sdist_only")
     expected.discard("graph_file")
+    # test_mode is not supported in bootstrap-parallel (serial mode only)
+    expected.discard("test_mode")
 
     assert set(get_option_names(bootstrap.bootstrap_parallel)) == expected