Ship linkable libraries in the macOS wheel

.ci/scripts/wheel/test_cpp_sdk.py

@@ -33,6 +33,7 @@
 _EXPORT_SCRIPT = """
 import json
 import sys
+from pathlib import Path
 
 import torch
 from executorch.exir import to_edge_transform_and_lower
@@ -64,7 +65,15 @@ def forward(self, x, image):
     # variants of the quantized operators with torch. Without it the export fails with
     # "Missing out variants: quantized_decomposed::quantize_per_tensor", because the
     # lowering step has no out variant to select.
-    import executorch.kernels.quantized  # noqa: F401
+    # Loaded directly rather than through executorch.kernels.quantized, whose __init__
+    # swallows every exception, so a load failure would otherwise appear much later as
+    # "Missing out variants" with no indication of why.
+    import executorch as _executorch
+
+    _root = Path(list(_executorch.__path__)[0]) / "kernels" / "quantized"
+    _libs = sorted(_root.glob("*quantized_ops_aot_lib.*"))
+    assert len(_libs) == 1, f"expected one ahead-of-time library, found {_libs}"
+    torch.ops.load_library(str(_libs[0]))
     from executorch.backends.xnnpack.quantizer.xnnpack_quantizer import (
         get_symmetric_quantization_config,
         XNNPACKQuantizer,
@@ -261,6 +270,38 @@ def _consumer_cmake(components) -> str:
 """
 
 
+def _dynamic_lib_suffix() -> str:
+    """The loadable library suffix on this platform, including the dot."""
+    return ".dylib" if sys.platform == "darwin" else ".so"
+
+
+def _library_file_name(base_name: str) -> str:
+    """The file name a library has on this platform."""
+    return f"{base_name}{_dynamic_lib_suffix()}"
+
+
+def _recorded_dependencies(binary) -> str:
+    """What a built binary records about its dependencies and search paths.
+
+    readelf prints the ELF dynamic section, otool -l the Mach-O load commands. Both
+    carry the same facts: a dependency entry and a runtime search path entry, named
+    NEEDED and RUNPATH on ELF, LC_LOAD_DYLIB and LC_RPATH on Mach-O.
+    """
+    if sys.platform == "darwin":
+        tool, args = _tool("otool"), ["-l"]
+        needed = "otool"
+    else:
+        tool, args = _tool("readelf"), ["-d"]
+        needed = "readelf"
+    assert tool is not None, f"{needed} is needed to read the runtime search path"
+    return subprocess.run(
+        [tool, *args, str(binary)],
+        capture_output=True,
+        text=True,
+        check=True,
+    ).stdout
+
+
 def _tool(name: str) -> str:
     """Locate a build tool, including one pip installed beside this interpreter.
 
@@ -560,14 +601,8 @@ def test_consumer_is_relocatable(work_dir: Path) -> None:
     model, reference = _export(work_dir, "plain")
     consumer = _build_consumer(work_dir, "relocate", ["runtime", "kernels_optimized"])
 
-    assert shutil.which("readelf") is not None, "readelf is needed to read the RUNPATH"
-    dynamic = subprocess.run(
-        [_tool("readelf"), "-d", str(consumer)],
-        capture_output=True,
-        text=True,
-        check=True,
-    ).stdout
-    assert "libexecutorch.so" in dynamic, (
+    dynamic = _recorded_dependencies(consumer)
+    assert _library_file_name("libexecutorch") in dynamic, (
         "the application records no dependency on the shipped runtime, so it is not "
         f"linking what the wheel ships:\n{dynamic}"
     )
@@ -591,7 +626,7 @@ def test_consumer_is_relocatable(work_dir: Path) -> None:
     deployed = work_dir / "deployed"
     deployed.mkdir(parents=True, exist_ok=True)
     shutil.copy2(consumer, deployed / "consumer")
-    for library in sorted((package_dir / "lib").glob("lib*.so*")):
+    for library in sorted((package_dir / "lib").glob(_library_file_name("lib*") + "*")):
         if library.is_file() and not library.is_symlink():
             shutil.copy2(library, deployed / library.name)
 
@@ -783,7 +818,9 @@ def test_profiler_component_is_usable(work_dir: Path) -> None:
     # Globbed, not an exact name: the library carries a version suffix outside a wheel build, and an exact
     # match would silently skip this check there. The profiler is required elsewhere in this suite, so its
     # absence is a fault rather than a reason to skip.
-    shipped = sorted((package_dir / "lib").glob("libexecutorch_etdump.so*"))
+    shipped = sorted(
+        (package_dir / "lib").glob(_library_file_name("libexecutorch_etdump") + "*")
+    )
     assert shipped, (
         f"the wheel ships no profiler library under {package_dir / 'lib'}, so the etdump component it "
         "advertises cannot be linked"
@@ -1019,7 +1056,7 @@ def test_shipped_headers_have_implementations(work_dir: Path) -> None:
                         "executorch_kernels_optimized",
                         "executorch_threadpool",
                     )
-                    if (library_dir / f"lib{name}.so").is_file()
+                    if (library_dir / (f"lib{name}" + _dynamic_lib_suffix())).is_file()
                 ],
                 f"-Wl,-rpath,{library_dir}",
             ],
@@ -1229,10 +1266,8 @@ def test_pre_3_28_route_builds_a_consumer_through_variables(work_dir: Path) -> N
     # the only thing that carries them on this route. Reading the dynamic section rather
     # than running because running needs a model, which the modern-CMake tests above
     # cover once and this one only owns the variables path.
-    dependencies = subprocess.run(
-        ["readelf", "-d", str(consumer)], capture_output=True, text=True, check=True
-    ).stdout
-    assert "libexecutorch.so" in dependencies, (
+    dependencies = _recorded_dependencies(consumer)
+    assert _library_file_name("libexecutorch") in dependencies, (
         "a consumer built through EXECUTORCH_LIBRARIES on pre-3.28 CMake does not "
         f"depend on the runtime:\n{dependencies}"
     )
@@ -1262,7 +1297,11 @@ def test_quantized_kernels_component_runs_a_model(work_dir: Path) -> None:
     package_dir = _installed_package_dir()
     # Globbed for the same reason the profiler check is: the library carries a version suffix outside a
     # wheel build, and an exact name would skip this silently there rather than running it.
-    shipped = sorted((package_dir / "lib").glob("libexecutorch_kernels_quantized.so*"))
+    shipped = sorted(
+        (package_dir / "lib").glob(
+            _library_file_name("libexecutorch_kernels_quantized") + "*"
+        )
+    )
     assert shipped, (
         "the wheel ships no quantized kernels library. The preset that builds it enables "
         "them unconditionally, so this is a packaging or build regression rather than an "
@@ -1311,7 +1350,9 @@ def test_aggregate_variable_excludes_the_quantized_kernels(work_dir: Path) -> No
     # always enables these kernels, so their absence is a regression rather than a
     # configuration to tolerate, and skipping would report this as coverage.
     assert sorted(
-        (package_dir / "lib").glob("libexecutorch_kernels_quantized.so*")
+        (package_dir / "lib").glob(
+            _library_file_name("libexecutorch_kernels_quantized") + "*"
+        )
     ), "the wheel ships no quantized kernels library, so this check cannot run"
 
     source_dir = work_dir / "aggregate-only"
@@ -1346,9 +1387,7 @@ def test_aggregate_variable_excludes_the_quantized_kernels(work_dir: Path) -> No
         )
 
     consumer = build_dir / "consumer"
-    dependencies = subprocess.run(
-        ["readelf", "-d", str(consumer)], capture_output=True, text=True, check=True
-    ).stdout
+    dependencies = _recorded_dependencies(consumer)
     assert "libexecutorch_kernels_quantized" not in dependencies, (
         "an application that linked only ${EXECUTORCH_LIBRARIES} depends on the "
         "quantized kernels. That library collides with the export-time plugin, so it "

.ci/scripts/wheel/test_macos.py

@@ -6,12 +6,30 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+import tempfile
+from pathlib import Path
+
 import test_base
+import test_cpp_sdk
+import test_shared_libraries
 from examples.models import Backend, Model
 
 if __name__ == "__main__":
     test_base.test_cmsis_nn_install()
 
+    # The wheel ships the runtime, the kernels, the delegate, the thread pool and
+    # the profiler as separate libraries here too, so check that each has exactly
+    # one owner and that all of them are loadable.
+    with tempfile.TemporaryDirectory() as work_dir:
+        test_shared_libraries.run_tests(Path(work_dir))
+
+    # And that a C++ application outside the wheel can actually use them. Nothing
+    # else covers this: the Python extension links those libraries itself, so it
+    # passes whether or not the package config names them or the shipped headers
+    # are complete.
+    with tempfile.TemporaryDirectory() as work_dir:
+        test_cpp_sdk.run_tests(Path(work_dir))
+
     test_base.run_tests(
         model_tests=[
             test_base.ModelTest(

CMakeLists.txt

@@ -193,17 +193,16 @@ if(DEFINED EXECUTORCH_BAREMETAL_SKIP_INSTALL
 endif()
 
 if(EXECUTORCH_BUILD_SHARED)
-  # Linux only, and said here rather than left to fail somewhere downstream. The
-  # shared build names libraries with an ELF soname, records $ORIGIN runtime
-  # paths, and uses GNU linker options to keep a registration-only library on a
-  # link line. None of that applies on Apple, which is served by the Swift
-  # package distribution, or on Windows, where the runtime carries no export
-  # annotations for a DLL. Enabling it elsewhere failed much later and less
-  # clearly, when packaging looked for a .so the build never emitted.
-  if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux")
+  # Said here rather than left to fail somewhere downstream, where packaging
+  # looked for a library the build never emitted. Windows is still refused:
+  # there the runtime carries no export annotations, so a DLL would link against
+  # nothing, which is a missing capability rather than a different spelling of
+  # one.
+  if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux" AND NOT APPLE)
     message(
-      FATAL_ERROR "EXECUTORCH_BUILD_SHARED is supported on Linux only, not "
-                  "${CMAKE_SYSTEM_NAME}."
+      FATAL_ERROR
+        "EXECUTORCH_BUILD_SHARED is supported on Linux and macOS only, not "
+        "${CMAKE_SYSTEM_NAME}."
     )
   endif()
   set(CMAKE_POSITION_INDEPENDENT_CODE ON)
@@ -1192,8 +1191,17 @@ if(EXECUTORCH_BUILD_PYBIND)
 
   # RPATH for _portable_lib.so. It sits in
   # <site-packages>/executorch/extension/pybindings, so torch is three levels up
-  # and the wheel's own lib/ directory is two.
-  set(_portable_lib_rpath "$ORIGIN/../../../torch/lib")
+  # and the wheel's own lib/ directory is two. Mach-O spells the loader relative
+  # token differently and takes a list rather than a colon joined string, so
+  # both differ here while the layout reasoning does not.
+  if(APPLE)
+    set(_portable_lib_origin "@loader_path")
+    set(_portable_lib_rpath_separator ";")
+  else()
+    set(_portable_lib_origin "$ORIGIN")
+    set(_portable_lib_rpath_separator ":")
+  endif()
+  set(_portable_lib_rpath "${_portable_lib_origin}/../../../torch/lib")
 
   if(EXECUTORCH_BUILD_EXTENSION_MODULE)
     # extension_module_static is already bundled into libexecutorch.so; linking
@@ -1243,12 +1251,20 @@ if(EXECUTORCH_BUILD_PYBIND)
   endif()
 
   if(EXECUTORCH_BUILD_CUDA)
-    string(APPEND _portable_lib_rpath ":$ORIGIN/../../backends/cuda")
+    string(
+      APPEND
+      _portable_lib_rpath
+      "${_portable_lib_rpath_separator}${_portable_lib_origin}/../../backends/cuda"
+    )
   endif()
 
   if(EXECUTORCH_BUILD_QNN)
     list(APPEND _dep_libs qnn_executorch_backend)
-    string(APPEND _portable_lib_rpath ":$ORIGIN/../../backends/qualcomm")
+    string(
+      APPEND
+      _portable_lib_rpath
+      "${_portable_lib_rpath_separator}${_portable_lib_origin}/../../backends/qualcomm"
+    )
   endif()
 
   if(EXECUTORCH_BUILD_ENN)

extension/llm/custom_ops/CMakeLists.txt

@@ -152,10 +152,13 @@ if(EXECUTORCH_BUILD_KERNELS_LLM_AOT)
     # to the shared build so a build that does not opt in keeps the parent's
     # install-only behaviour.
     set_target_properties(
-      custom_ops_aot_lib PROPERTIES BUILD_RPATH ${RPATH} INSTALL_RPATH ${RPATH}
+      custom_ops_aot_lib PROPERTIES BUILD_RPATH "${RPATH}" INSTALL_RPATH
+                                                           "${RPATH}"
     )
   else()
-    set_target_properties(custom_ops_aot_lib PROPERTIES INSTALL_RPATH ${RPATH})
+    set_target_properties(
+      custom_ops_aot_lib PROPERTIES INSTALL_RPATH "${RPATH}"
+    )
   endif()
   executorch_target_shared_runtime_path(
     custom_ops_aot_lib "extension/llm/custom_ops"

extension/training/CMakeLists.txt

@@ -108,12 +108,17 @@ if(EXECUTORCH_BUILD_PYBIND)
   endif()
   executorch_target_link_shared_runtime(_training_lib)
 
-  if(EXECUTORCH_BUILD_SHARED AND NOT APPLE)
+  if(EXECUTORCH_BUILD_SHARED)
     # This module links Torch directly, and the only other entry reaching it is
     # the absolute build directory CMake adds, which does not exist anywhere
     # else, so the Torch path is recorded here rather than left implicit.
+    if(APPLE)
+      set(_training_torch_path "@loader_path/../../../../torch/lib")
+    else()
+      set(_training_torch_path "$ORIGIN/../../../../torch/lib")
+    endif()
     set_target_properties(
-      _training_lib PROPERTIES INSTALL_RPATH "$ORIGIN/../../../../torch/lib"
+      _training_lib PROPERTIES INSTALL_RPATH "${_training_torch_path}"
     )
     executorch_target_shared_runtime_path(
       _training_lib "extension/training/pybindings"

kernels/portable/CMakeLists.txt

@@ -144,7 +144,7 @@ if(EXECUTORCH_BUILD_KERNELS_CUSTOM_AOT AND NOT EXECUTORCH_BUILD_ARM_BAREMETAL)
     set(RPATH "\$ORIGIN/../../extensions/pybindings")
   endif()
   set_target_properties(
-    portable_custom_ops_aot_lib PROPERTIES BUILD_RPATH ${RPATH} INSTALL_RPATH
-                                                                ${RPATH}
+    portable_custom_ops_aot_lib PROPERTIES BUILD_RPATH "${RPATH}" INSTALL_RPATH
+                                                                  "${RPATH}"
   )
 endif()

kernels/quantized/CMakeLists.txt

@@ -146,11 +146,22 @@ if(NOT CMAKE_GENERATOR STREQUAL "Xcode"
       # route to the runtime, and overwriting the property here would drop it.
       get_target_property(_existing quantized_ops_aot_lib INSTALL_RPATH)
       if(_existing)
-        set(RPATH "${_existing}:${RPATH}")
+        # Mach-O keeps one entry per load command, so the two are a CMake list
+        # there. Joining them with a colon produced a single unusable path
+        # containing both, and the library then found neither the runtime nor
+        # the extension.
+        if(APPLE)
+          set(RPATH "${_existing};${RPATH}")
+        else()
+          set(RPATH "${_existing}:${RPATH}")
+        endif()
       endif()
+      # Quoted, because on Apple this is a list: unquoted it expands into
+      # separate arguments and the trailing entries are read as further property
+      # keywords, leaving the search path empty.
       set_target_properties(
-        quantized_ops_aot_lib PROPERTIES BUILD_RPATH ${RPATH} INSTALL_RPATH
-                                                              ${RPATH}
+        quantized_ops_aot_lib PROPERTIES BUILD_RPATH "${RPATH}" INSTALL_RPATH
+                                                                "${RPATH}"
       )
     endif()
   endif()