feat: Add --test-mode for resilient bootstrap with failure handling

Add --test-mode flag that enables resilient bootstrapping by marking failed
packages as pre-built and continuing until all packages are processed. Uses
optimal n+1 retry logic with comprehensive failure reporting including exception
types, messages, and per-package context.

Major changes:
- Enhanced BuildResult dataclass with req, resolved_version, and exception
  tracking for detailed failure analysis
- Refactored pre_built_override from Settings to WorkContext for proper
  separation of static config vs runtime state
- Introduced public WorkContext.package_build_info() API, replacing direct
  Settings access across commands (bootstrap, build, graph, list-overrides)
- Fixed build-parallel command to use new public API
- Added 4 essential test scenarios in test_bootstrap_test_mode.py

Benefits:
- Discover all build failures in one run rather than stopping on first failure
- Support mixed source/binary dependency workflows
- Better error context for debugging failed builds
- Cleaner API boundaries between configuration and runtime context

Fixes #713

Co-developed-with: Cursor IDE with Claude 4.0 Sonnet

Signed-off-by: Lalatendu Mohanty <[email protected]>

Author: LalatenduMohanty

src/fromager/bootstrapper.py

@@ -97,6 +97,13 @@ def _get_requirements_from_args(
     default=False,
     help="Skip generating constraints.txt file to allow building collections with conflicting versions",
 )
+@click.option(
+    "--test-mode",
+    "test_mode",
+    is_flag=True,
+    default=False,
+    help="Test mode: mark failed packages as pre-built and continue, report failures at end",
+)
 @click.argument("toplevel", nargs=-1)
 @click.pass_obj
 def bootstrap(
@@ -106,6 +113,7 @@ def bootstrap(
     cache_wheel_server_url: str | None,
     sdist_only: bool,
     skip_constraints: bool,
+    test_mode: bool,
     toplevel: list[str],
 ) -> None:
     """Compute and build the dependencies of a set of requirements recursively
@@ -116,6 +124,11 @@ def bootstrap(
     """
     logger.info(f"cache wheel server url: {cache_wheel_server_url}")
 
+    if test_mode:
+        logger.info(
+            "test mode enabled: will mark failed packages as pre-built and continue"
+        )
+
     to_build = _get_requirements_from_args(toplevel, requirements_files)
     if not to_build:
         raise RuntimeError(
@@ -148,6 +161,7 @@ def bootstrap(
             prev_graph,
             cache_wheel_server_url,
             sdist_only=sdist_only,
+            test_mode=test_mode,
         )
 
         # we need to resolve all the top level dependencies before we start bootstrapping.
@@ -183,9 +197,26 @@ def bootstrap(
 
         for req in to_build:
             token = requirement_ctxvar.set(req)
-            bt.bootstrap(req, requirements_file.RequirementType.TOP_LEVEL)
-            progressbar.update()
-            requirement_ctxvar.reset(token)
+            try:
+                bt.bootstrap(req, requirements_file.RequirementType.TOP_LEVEL)
+                progressbar.update()
+                if test_mode:
+                    logger.info("Successfully processed: %s", req)
+            except Exception as err:
+                if test_mode:
+                    # Test mode: log error but continue processing
+                    logger.error(
+                        "test mode: failed to process %s: %s",
+                        req,
+                        err,
+                        exc_info=True,  # Full traceback to debug log
+                    )
+                    progressbar.update()  # Update progress even on failure
+                else:
+                    # Normal mode: re-raise the exception (fail-fast)
+                    raise
+            finally:
+                requirement_ctxvar.reset(token)
 
     constraints_filename = wkctx.work_dir / "constraints.txt"
     if skip_constraints:
@@ -200,7 +231,57 @@ def bootstrap(
 
     logger.debug("match_py_req LRU cache: %r", resolver.match_py_req.cache_info())
 
-    metrics.summarize(wkctx, "Bootstrapping")
+    # Test mode summary reporting
+    if test_mode:
+        if bt.failed_builds:
+            # Use repository's logging pattern for error reporting
+            logger.error("test mode: the following packages failed to build:")
+            for failure in sorted(
+                bt.failed_builds, key=lambda f: str(f.req) if f.req else ""
+            ):
+                if failure.req and failure.resolved_version:
+                    logger.error(
+                        "  - %s==%s",
+                        failure.req,
+                        failure.resolved_version,
+                    )
+                    if failure.exception_type:
+                        logger.error(
+                            "    Error: %s: %s",
+                            failure.exception_type,
+                            failure.exception_message,
+                        )
+                else:
+                    logger.error("  - unknown package (missing context)")
+
+            # Categorize failures by exception type for better analysis
+            failure_types: dict[str, list[str]] = {}
+            for failure in bt.failed_builds:
+                exc_type = failure.exception_type or "Unknown"
+                pkg_name = (
+                    f"{failure.req}=={failure.resolved_version}"
+                    if failure.req and failure.resolved_version
+                    else "unknown"
+                )
+                failure_types.setdefault(exc_type, []).append(pkg_name)
+
+            logger.error("")
+            logger.error("test mode: failure breakdown by type:")
+            for exc_type, packages in sorted(failure_types.items()):
+                logger.error("  %s: %d package(s)", exc_type, len(packages))
+
+            logger.error(
+                "test mode: %d package(s) failed to build", len(bt.failed_builds)
+            )
+            # Follow repository's error exit pattern like __main__.py and lint.py
+            raise SystemExit(
+                f"Test mode completed with {len(bt.failed_builds)} build failures"
+            )
+        else:
+            logger.info("test mode: all packages built successfully")
+        metrics.summarize(wkctx, "Test Mode Bootstrapping")
+    else:
+        metrics.summarize(wkctx, "Bootstrapping")
 
 
 def write_constraints_file(
@@ -480,6 +561,9 @@ def bootstrap_parallel(
     remaining wheels in parallel. The bootstrap step downloads sdists
     and builds build-time dependency in serial. The build-parallel step
     builds the remaining wheels in parallel.
+
+    Note: --test-mode is not supported with bootstrap-parallel. Use
+    'bootstrap --test-mode' for comprehensive failure testing.
     """
     # Do not remove build environments in bootstrap phase to speed up the
     # parallel build phase.
@@ -495,6 +579,7 @@ def bootstrap_parallel(
         cache_wheel_server_url=cache_wheel_server_url,
         sdist_only=True,
         skip_constraints=skip_constraints,
+        test_mode=False,
         toplevel=toplevel,
     )
 

src/fromager/commands/bootstrap.py

@@ -675,9 +675,7 @@ def update_progressbar_cb(future: concurrent.futures.Future) -> None:
             exclusive_nodes: DependencyNodeList = [
                 node
                 for node in buildable_nodes
-                if wkctx.settings.package_build_info(
-                    node.canonicalized_name
-                ).exclusive_build
+                if wkctx.package_build_info(node.canonicalized_name).exclusive_build
             ]
             if exclusive_nodes:
                 # Only build the first exclusive node this round

src/fromager/commands/build.py

@@ -125,7 +125,7 @@ def _find_customized_nodes(
     """Filter nodes to find only those with customizations."""
     customized_nodes: list[DependencyNode] = []
     for node in nodes:
-        pbi = wkctx.settings.package_build_info(node.canonicalized_name)
+        pbi = wkctx.package_build_info(node.canonicalized_name)
         if node.canonicalized_name != ROOT and pbi.has_customizations:
             customized_nodes.append(node)
     return customized_nodes
@@ -161,7 +161,7 @@ def _find_customized_dependencies_for_node(
                 continue
 
             child = edge.destination_node
-            child_pbi = wkctx.settings.package_build_info(child.canonicalized_name)
+            child_pbi = wkctx.package_build_info(child.canonicalized_name)
             new_path = path + [current_node.key]
 
             # Use the first requirement we encounter in the path
@@ -277,7 +277,7 @@ def get_node_id(node: str) -> str:
         if not name:
             node_type.append("toplevel")
         else:
-            pbi = wkctx.settings.package_build_info(name)
+            pbi = wkctx.package_build_info(name)
             all_patches: PatchMap = pbi.get_all_patches()
 
             if node.pre_built:

src/fromager/commands/graph.py

@@ -65,7 +65,7 @@ def list_overrides(
     export_data = []
 
     for name in overridden_packages:
-        pbi = wkctx.settings.package_build_info(name)
+        pbi = wkctx.package_build_info(name)
         ps = wkctx.settings.package_setting(name)
 
         plugin_hooks: list[str] = []

src/fromager/commands/list_overrides.py

@@ -95,9 +95,19 @@ def __init__(
 
         self._parallel_builds = False
 
+        self._pre_built_override: set[NormalizedName] = set()
+
     def enable_parallel_builds(self) -> None:
         self._parallel_builds = True
 
+    def add_pre_built_override(self, package_name: str | NormalizedName) -> None:
+        """Mark a package as pre-built at runtime."""
+        self._pre_built_override.add(canonicalize_name(package_name))
+
+    def is_pre_built_override(self, package_name: str | NormalizedName) -> bool:
+        """Check if a package is marked as pre-built override."""
+        return canonicalize_name(package_name) in self._pre_built_override
+
     @property
     def wheels_build(self) -> pathlib.Path:
         # when parallel builds are enabled, return a path that is unique for the
@@ -164,7 +174,7 @@ def package_build_info(
             name = package.name
         else:
             name = package
-        return self.settings.package_build_info(name)
+        return self.settings.package_build_info(name, self)
 
     def setup(self) -> None:
         # The work dir must already exist, so don't try to create it.

src/fromager/context.py

@@ -624,12 +624,26 @@ def get_available_memory_gib() -> float:
 
 
 class PackageBuildInfo:
-    """Package build information
+    """Variant-aware package build configuration and metadata.
 
-    Public API for PackageSettings with i
+    Primary public API for accessing package-specific settings during the build
+    process. Combines static configuration from YAML files with runtime context
+    to provide variant-specific (cpu, cuda, etc.) build information.
+
+    Key responsibilities:
+    - Determine if package should be built or use pre-built wheels
+    - Provide patches to apply for specific versions
+    - Configure build environment (parallel jobs, environment variables)
+    - Manage package customizations (plugins, custom download URLs)
+    - Calculate build tags from changelogs for wheel versioning
+
+    Instances are cached per package and accessed via ``WorkContext.package_build_info()``.
     """
 
-    def __init__(self, settings: Settings, ps: PackageSettings) -> None:
+    def __init__(
+        self, settings: Settings, ps: PackageSettings, ctx: context.WorkContext
+    ) -> None:
+        self._ctx = ctx
         self._variant = typing.cast(Variant, settings.variant)
         self._patches_dir = settings.patches_dir
         self._variant_changelog = settings.variant_changelog()
@@ -745,6 +759,11 @@ def has_customizations(self) -> bool:
     @property
     def pre_built(self) -> bool:
         """Does the variant use pre-build wheels?"""
+        # Check if package is in runtime pre_built_override
+        if self._ctx.is_pre_built_override(self.package):
+            return True
+
+        # Check variant configuration
         vi = self._ps.variants.get(self.variant)
         if vi is not None:
             return vi.pre_built
@@ -1146,23 +1165,27 @@ def package_setting(self, package: str | Package) -> PackageSettings:
             self._package_settings[package] = ps
         return ps
 
-    def package_build_info(self, package: str | Package) -> PackageBuildInfo:
+    def package_build_info(
+        self, package: str | Package, ctx: context.WorkContext
+    ) -> PackageBuildInfo:
         """Get (cached) PackageBuildInfo for package and current variant"""
         package = Package(canonicalize_name(package, validate=True))
         pbi = self._pbi_cache.get(package)
         if pbi is None:
             ps = self.package_setting(package)
-            pbi = PackageBuildInfo(self, ps)
+            pbi = PackageBuildInfo(self, ps, ctx)
             self._pbi_cache[package] = pbi
         return pbi
 
     def list_pre_built(self) -> set[Package]:
-        """List packages marked as pre-built"""
-        return set(
-            name
-            for name in self._package_settings
-            if self.package_build_info(name).pre_built
-        )
+        """List packages marked as pre-built by configuration"""
+        result = set()
+        for name in self._package_settings:
+            ps = self._package_settings[name]
+            vi = ps.variants.get(self._variant)
+            if vi is not None and vi.pre_built:
+                result.add(name)
+        return result
 
     def list_overrides(self) -> set[Package]:
         """List packages with overrides