diff --git a/.github/workflows/_build.yml b/.github/workflows/_build.yml index da029f210..b77e505a7 100644 --- a/.github/workflows/_build.yml +++ b/.github/workflows/_build.yml @@ -429,6 +429,31 @@ jobs: - name: Build package (Wheel Only) run: uv build --wheel + - name: Smoke test built wheel (Windows) + if: runner.os == 'Windows' + shell: bash + run: | + python -m pip install --upgrade pip + python -m pip install --force-reinstall dist/*.whl + + cd "$RUNNER_TEMP" + python - <<'PY' + import importlib.util + + import openviking.storage.vectordb.engine as engine + + print(f"Loaded runtime engine variant {engine.ENGINE_VARIANT}") + print(f"Available engine variants {engine.AVAILABLE_ENGINE_VARIANTS}") + + module_name = f"openviking.storage.vectordb.engine._{engine.ENGINE_VARIANT}" + backend_spec = importlib.util.find_spec(module_name) + if backend_spec is None or backend_spec.origin is None: + raise SystemExit(f"backend module {module_name} was not installed") + + print(f"Imported backend module {module_name}") + print(f"Backend module origin {backend_spec.origin}") + PY + - name: Store the distribution packages uses: actions/upload-artifact@v7 with: diff --git a/openviking/storage/vectordb/engine/__init__.py b/openviking/storage/vectordb/engine/__init__.py index f87b45e82..843103e92 100644 --- a/openviking/storage/vectordb/engine/__init__.py +++ b/openviking/storage/vectordb/engine/__init__.py @@ -27,6 +27,7 @@ "avx2": "x86_avx2", "avx512": "x86_avx512", } +_WINDOWS_DLL_DIR_HANDLES = [] def _is_x86_machine(machine: str | None = None) -> bool: @@ -143,6 +144,25 @@ def _load_backend(variant: str) -> ModuleType: return importlib.import_module(f".{module_name}", __name__) +def _register_windows_dll_dirs(module_path: Path) -> None: + if sys.platform != "win32" or not hasattr(os, "add_dll_directory"): + return + + package_root = module_path.parents[2] + search_dirs = [ + module_path, + package_root / "lib", + package_root / "bin", + ] + seen = set() + for search_dir in search_dirs: + resolved = search_dir.resolve() + if resolved in seen or not resolved.exists(): + continue + seen.add(resolved) + _WINDOWS_DLL_DIR_HANDLES.append(os.add_dll_directory(str(resolved))) + + def _export_backend(module: ModuleType) -> tuple[str, ...]: if getattr(module, "_ENGINE_BACKEND_API", None) == "abi3-v1": exports = build_abi3_exports(module) @@ -185,6 +205,7 @@ def __repr__(self) -> str: _EXPORTED_NAMES = () else: ENGINE_VARIANT = _SELECTED_VARIANT + _register_windows_dll_dirs(Path(__file__).resolve().parent) _BACKEND = _load_backend(ENGINE_VARIANT) _EXPORTED_NAMES = _export_backend(_BACKEND) diff --git a/setup.py b/setup.py index 91be3535b..f7baaf65c 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,41 @@ ENGINE_BUILD_CONFIG = get_host_engine_build_config(platform.machine()) +def _get_windows_python_sabi_library() -> Path: + """Return the stable-ABI Python library path for Windows abi3 extensions.""" + candidate_roots = [] + for raw_root in ( + sys.base_prefix, + sys.base_exec_prefix, + sysconfig.get_config_var("installed_base"), + sysconfig.get_config_var("base"), + ): + if not raw_root: + continue + candidate_root = Path(raw_root).resolve() + if candidate_root not in candidate_roots: + candidate_roots.append(candidate_root) + + candidate_paths = [] + for root in candidate_roots: + candidate_paths.extend( + [ + root / "libs" / "python3.lib", + root / "python3.dll", + ] + ) + + for candidate_path in candidate_paths: + if candidate_path.exists(): + return candidate_path + + searched = ", ".join(str(path) for path in candidate_paths) or "" + raise RuntimeError( + "Could not locate the Windows stable-ABI Python library for abi3 engine modules. " + f"Searched: {searched}" + ) + + class OpenVikingBuildExt(build_ext): """Build OpenViking runtime artifacts and Python native extensions.""" @@ -397,6 +432,8 @@ def _build_extension_impl(self, ext_fullpath, ext_dir, build_dir): if target_arch: cmake_args.append(f"-DCMAKE_OSX_ARCHITECTURES={target_arch}") elif sys.platform == "win32": + windows_python_sabi_library = _get_windows_python_sabi_library() + cmake_args.append(f"-DOV_PYTHON_SABI_LIBRARY={windows_python_sabi_library}") cmake_args.extend(["-G", "MinGW Makefiles"]) self.spawn([self.cmake_executable] + cmake_args) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ed1663e75..1a79b8a43 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -8,6 +8,7 @@ include(CMakeParseArguments) set(OV_X86_BUILD_VARIANTS "sse3;avx2;avx512" CACHE STRING "x86 engine variants to build") set(OV_PY_OUTPUT_DIR "" CACHE PATH "Output directory for Python extension modules") set(OV_PY_EXT_SUFFIX ".so" CACHE STRING "Python extension suffix, including ABI tag if needed") +set(OV_PYTHON_SABI_LIBRARY "" CACHE FILEPATH "Stable-ABI Python import library or DLL for Windows abi3 modules") if(NOT OV_PY_OUTPUT_DIR) set(OV_PY_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/python_engine") @@ -43,6 +44,9 @@ set(CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE} -lpthread") set(Python3_ARCH_INCLUDE_DIR "/usr/include/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/") find_package(Python3 COMPONENTS Interpreter Development REQUIRED) +if(WIN32 AND CMAKE_VERSION VERSION_GREATER_EQUAL "3.26") + find_package(Python3 COMPONENTS Development.SABIModule QUIET) +endif() if(UNIX AND NOT APPLE) set(Python3_LIBRARIES "") @@ -183,6 +187,25 @@ function(ov_get_x86_variant_flags variant out_flags out_defs out_supported) set(${out_supported} ${OV_SUPPORTED} PARENT_SCOPE) endfunction() +function(ov_link_python_abi target_name) + if(WIN32) + if(OV_PYTHON_SABI_LIBRARY) + target_link_libraries(${target_name} PRIVATE "${OV_PYTHON_SABI_LIBRARY}") + elseif(TARGET Python3::SABIModule) + target_link_libraries(${target_name} PRIVATE Python3::SABIModule) + else() + message( + FATAL_ERROR + "Windows abi3 modules require OV_PYTHON_SABI_LIBRARY or Python3::SABIModule" + ) + endif() + endif() + + if(APPLE) + target_link_options(${target_name} PRIVATE "-undefined" "dynamic_lookup") + endif() +endfunction() + function(ov_add_python_backend backend_suffix module_name) set(oneValueArgs INDEX_LIBRARY) set(multiValueArgs COMPILE_OPTIONS COMPILE_DEFINITIONS) @@ -207,9 +230,7 @@ function(ov_add_python_backend backend_suffix module_name) ${OV_BACKEND_INDEX_LIBRARY} Threads::Threads ) - if(WIN32 AND TARGET Python3::Python) - target_link_libraries(${MODULE_TARGET} PRIVATE Python3::Python) - endif() + ov_link_python_abi(${MODULE_TARGET}) ov_link_filesystem_libs(${MODULE_TARGET}) if(MINGW) @@ -232,10 +253,6 @@ function(ov_add_python_backend backend_suffix module_name) OUTPUT_NAME "${module_name}" SUFFIX "${OV_PY_EXT_SUFFIX}" ) - - if(APPLE) - target_link_options(${MODULE_TARGET} PRIVATE "-undefined" "dynamic_lookup") - endif() endfunction() set(OV_ENGINE_IMPL_TARGET "") @@ -270,9 +287,7 @@ if(OV_PLATFORM_X86) add_library(engine_module_x86_caps MODULE abi3_x86_caps.cpp) target_include_directories(engine_module_x86_caps PRIVATE ${Python3_INCLUDE_DIRS}) target_compile_definitions(engine_module_x86_caps PRIVATE Py_LIMITED_API=0x030A0000) - if(WIN32 AND TARGET Python3::Python) - target_link_libraries(engine_module_x86_caps PRIVATE Python3::Python) - endif() + ov_link_python_abi(engine_module_x86_caps) set_target_properties( engine_module_x86_caps PROPERTIES @@ -282,9 +297,6 @@ if(OV_PLATFORM_X86) OUTPUT_NAME "_x86_caps" SUFFIX "${OV_PY_EXT_SUFFIX}" ) - if(APPLE) - target_link_options(engine_module_x86_caps PRIVATE "-undefined" "dynamic_lookup") - endif() if(TARGET engine_index_sse3) add_library(engine_impl INTERFACE) diff --git a/tests/misc/test_abi3_packaging_config.py b/tests/misc/test_abi3_packaging_config.py index 2638c1c8b..3df323254 100644 --- a/tests/misc/test_abi3_packaging_config.py +++ b/tests/misc/test_abi3_packaging_config.py @@ -13,10 +13,28 @@ def test_packaging_only_includes_abi3_engine_extensions(): assert "storage/vectordb/engine/*.abi3.so" in setup_py assert "storage/vectordb/engine/*.abi3.so" in pyproject + assert "storage/vectordb/engine/*.dll" not in setup_py + assert "storage/vectordb/engine/*.dll" not in pyproject assert "storage/vectordb/engine/*.so" not in setup_py assert "storage/vectordb/engine/*.so" not in pyproject +def test_windows_engine_loader_registers_dll_search_paths(): + engine_init = _read_text("openviking/storage/vectordb/engine/__init__.py") + + assert "add_dll_directory" in engine_init + assert "module_path" in engine_init + assert 'package_root / "lib"' in engine_init + assert 'package_root / "bin"' in engine_init + + +def test_setup_no_longer_bundles_mingw_runtime_dlls_for_engine(): + setup_py = _read_text("setup.py") + + assert "WINDOWS_ENGINE_RUNTIME_DLL_PATTERNS" not in setup_py + assert "_stage_windows_engine_runtime_dlls" not in setup_py + + def test_release_workflows_default_to_single_cp310_and_drop_pybind11(): build_workflow = _read_text(".github/workflows/_build.yml") release_workflow = _read_text(".github/workflows/release.yml") @@ -46,6 +64,33 @@ def test_release_build_workflow_no_longer_defines_extra_wheel_verify_jobs(): assert "verify-macos-14-wheel-on-macos-15:" not in build_workflow +def test_build_workflow_smoke_tests_windows_wheel_engine_import(): + build_workflow = _read_text(".github/workflows/_build.yml") + + assert "Smoke test built wheel (Windows)" in build_workflow + assert "python -m pip install --force-reinstall dist/*.whl" in build_workflow + assert 'cd "$RUNNER_TEMP"' in build_workflow + assert "import openviking.storage.vectordb.engine as engine" in build_workflow + assert "engine.ENGINE_VARIANT" in build_workflow + + +def test_windows_abi3_backend_uses_stable_python_linkage(): + setup_py = _read_text("setup.py") + src_cmake = _read_text("src/CMakeLists.txt") + + assert "OV_PYTHON_SABI_LIBRARY" in setup_py + assert "python3.dll" in setup_py + assert "OV_PYTHON_SABI_LIBRARY" in src_cmake + assert "Python3::Python" not in src_cmake + + +def test_build_workflow_no_longer_defines_windows_python312_verify_job(): + build_workflow = _read_text(".github/workflows/_build.yml") + + assert "verify-windows-abi3-wheel-on-python312:" not in build_workflow + assert "Smoke test Windows abi3 wheel on Python 3.12" not in build_workflow + + def test_abi3_backend_releases_gil_and_rejects_invalid_storage_op_type(): backend_source = _read_text("src/abi3_engine_backend.cpp")