Optimizing Pybind11 Development Workflow with scikit-build

[brupelo]

I'm transitioning from an old workflow based on CMakeLists.txt and setup.py to a more modern, cross-platform setup using scikit-build, pyproject.toml, and uv. Here's what

I’ve put together so far:

CMakeLists.txt:

cmake_minimum_required(VERSION 3.18)

# Set the project name first
project(hello_pybind11 LANGUAGES CXX)

# Enable faster builds with optimizations
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)

# Use Release builds by default for better performance
if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
    set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Build type" FORCE)
    set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
endif()

# Enable faster incremental builds
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)

# Use ccache if available for faster rebuilds
find_program(CCACHE_PROGRAM ccache)
if(CCACHE_PROGRAM)
    set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_PROGRAM}")
    set(CMAKE_C_COMPILER_LAUNCHER "${CCACHE_PROGRAM}")
    message(STATUS "Using ccache: ${CCACHE_PROGRAM}")
endif()

# Find pybind11 - use quiet mode to reduce output
find_package(pybind11 REQUIRED QUIET)

# Add the module
pybind11_add_module(hello_pybind11 main.cpp)

# Set version info properly
if(NOT DEFINED EXAMPLE_VERSION_INFO OR EXAMPLE_VERSION_INFO STREQUAL "")
    set(EXAMPLE_VERSION_INFO "0.1.0")
endif()

target_compile_definitions(hello_pybind11 PRIVATE VERSION_INFO=${EXAMPLE_VERSION_INFO})

# Cross-platform parallel compilation and optimizations
if(MSVC)
    # Windows: Enable parallel compilation and faster linking
    target_compile_options(hello_pybind11 PRIVATE
        /MP          # Multi-processor compilation
        /bigobj      # Large object files
    )
    # Use faster linker if available
    if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.15")
        set_target_properties(hello_pybind11 PROPERTIES
            MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>DLL"
        )
    endif()
elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
    # Linux/macOS: Enable optimizations
    target_compile_options(hello_pybind11 PRIVATE
        -Wall -Wextra
        $<$<CONFIG:Debug>:-g -O0>
        $<$<CONFIG:Release>:-O3 -DNDEBUG>
    )
    # Use faster linker if available
    if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
        # Try to use lld if available
        find_program(LLD_PROGRAM lld)
        if(LLD_PROGRAM)
            target_link_options(hello_pybind11 PRIVATE -fuse-ld=lld)
        endif()
    endif()
endif()

# Platform-specific settings
if(WIN32)
    target_compile_definitions(hello_pybind11 PRIVATE _WIN32_WINNT=0x0601)
elseif(APPLE)
    set_target_properties(hello_pybind11 PROPERTIES
        MACOSX_RPATH ON
        INSTALL_RPATH_USE_LINK_PATH ON
    )
elseif(UNIX)
    set_target_properties(hello_pybind11 PROPERTIES
        INSTALL_RPATH_USE_LINK_PATH ON
    )
endif()

# Enable detailed error messages only in debug mode
if(CMAKE_BUILD_TYPE STREQUAL "Debug")
    target_compile_definitions(hello_pybind11 PRIVATE PYBIND11_DETAILED_ERROR_MESSAGES)
endif()

# Install target
install(TARGETS hello_pybind11 DESTINATION .)

Justfile:

default:
    @just --list

setup:
    @echo "🚀 Setting up development environment..."
    uv venv
    uv pip install build scikit-build-core pybind11
    @echo "✅ Initial development install..."
    uv pip install -e . --config-settings=build-dir=build
    @echo "🎉 Setup complete! Use 'just dev' for fast iterations."

pre-dev:
    @echo "⚡ Reconfiguring CMake with venv settings..."
    @uv run cmake -S . -B build \
        -Dpybind11_DIR=`uv run python -c "import pybind11; print(pybind11.get_cmake_dir())"` \
        -DPython_EXECUTABLE=`uv run python -c "import sys; print(sys.executable)"` \
        -DCMAKE_BUILD_TYPE=Release \

dev:
    @echo "⚡ Building incrementally..."
    @uv run cmake --build build --config Release -j 4
    @echo "📦 Copying built module to site-packages..."
    @uv run python -c "import shutil, os, site; shutil.copy2(os.path.join('.venv', 'Release', 'hello_pybind11.cp311-win_amd64.pyd'), site.getsitepackages()[0])"

check:
    @echo "⚡ Quick import test..."
    @uv run python -c "import hello_pybind11; print(hello_pybind11.hello_world()); print(f'5 + 3 = {hello_pybind11.add(5, 3)}')"

test file="test.py":
    @echo "🧪 Running {{file}}..."
    @uv run python {{file}}

rebuild:
    @echo "🔄 Full rebuild..."
    @python -c "import shutil, os; [shutil.rmtree(p, ignore_errors=True) for p in ['build'] if os.path.exists(p)]"
    @python -c "import glob, os; [os.remove(f) for f in glob.glob('*.pyd') + glob.glob('*.so')]"
    @python -c "import shutil, glob; [shutil.rmtree(p, ignore_errors=True) for p in glob.glob('*.egg-info')]"
    uv pip install -e . --config-settings=build-dir=build --force-reinstall --no-deps

clean:
    @echo "🧹 Cleaning..."
    @python -c "import shutil, os; [shutil.rmtree(p, ignore_errors=True) for p in ['build', 'dist', '.venv'] if os.path.exists(p)]"
    @python -c "import glob, os; [os.remove(f) for f in glob.glob('*.pyd') + glob.glob('*.so')]"
    @python -c "import shutil, glob; [shutil.rmtree(p, ignore_errors=True) for p in glob.glob('*.egg-info')]"

info:
    @echo "📊 Build info:"
    @echo "pybind11 CMake Dir: `uv run python -c "import pybind11; print(pybind11.get_cmake_dir())"`"
    @echo "Python Executable: `uv run python -c "import sys; print(sys.executable)"`"
    @python -c "import os; print(f'Build dir: {os.path.abspath(\"build\")}')"
    @python -c "import sys; print(f'Python: {sys.executable}')"
    @python -c "import shutil; print(f'CMake: {\"found\" if shutil.which(\"cmake\") else \"not found\"}')"

profile:
    @echo "⏱️  Profiling build time..."
    @python -c "import time; start=time.time(); print(f'Starting build at: {time.strftime(\"%H:%M:%S\")}')"
    @python -c "import subprocess, time; start=time.time(); subprocess.run(['just', 'rebuild'], check=True, shell=True); print(f'Full rebuild time: {time.time()-start:.2f}s')"
    @python -c "import subprocess, time; start=time.time(); subprocess.run(['just', 'dev'], check=True, shell=True); print(f'Incremental dev time: {time.time()-start:.2f}s')"

main.cpp:

#include <pybind11/pybind11.h>
#include <string>

// Define the MACRO_STRINGIFY macro - handle empty values
#define STRINGIFY_HELPER(x) #x
#define STRINGIFY(x) STRINGIFY_HELPER(x)

std::string hello_world() {
    return "Hello, World from C++!";
}

int add(int a, int b) {
    return a + b;
}

int sub(int a, int b) {
    return a - b;
}

namespace py = pybind11;

PYBIND11_MODULE(hello_pybind11, m) {
    m.doc() = "pybind11 hello world plugin";

    m.def("hello_world", &hello_world, "A function that returns a greeting");
    m.def("add", &add, "A function that adds two numbers");
    m.def("sub", &sub, "A function that subtracts two numbers");

    // Set version - handle empty VERSION_INFO gracefully
    #if defined(VERSION_INFO)
        m.attr("__version__") = STRINGIFY(VERSION_INFO);
    #else
        m.attr("__version__") = "dev";
    #endif
}

pyproject.toml:

[build-system]
requires = ["scikit-build-core", "pybind11"]
build-backend = "scikit_build_core.build"

[project]
name = "hello-pybind11"
version = "0.1.0"
description = "A simple hello world example using pybind11"
authors = [{name = "Your Name", email = "your.email@example.com"}]
dependencies = []
requires-python = ">=3.8"

[tool.scikit-build]
# Use Release builds for better performance
cmake.build-type = "Release"
build.verbose = true        # Enable verbose output for debugging
wheel.expand-macos-universal-tags = true
build-dir = "build"

[tool.scikit-build.cmake.define]
PYBIND11_FINDPYTHON = "ON"

test.py:

#!/usr/bin/env python3
"""Test script for hello_pybind11 module"""

try:
    import hello_pybind11
    
    print("Testing hello_pybind11 module...")
    print(f"hello_world(): {hello_pybind11.hello_world()}")
    print(f"add(5, 3): {hello_pybind11.add(5, 3)}")
    print(f"sub(5, 3): {hello_pybind11.sub(5, 3)}")
    print(f"Version: {hello_pybind11.__version__}")
    print("All tests passed!")
    
except ImportError as e:
    print(f"Failed to import hello_pybind11: {e}")
    print("Make sure to build and install the module first")
except Exception as e:
    print(f"Error: {e}")

just setup, just clean, just rebuild recipes are ok... but the overhead and iteration times are quite slow, specially if the bootstrapping has already been made, for instance, trying to run just rebuild check it takes

ptime just rebuild check
🔄 Full rebuild...
←[1muv pip install -e . --config-settings=build-dir=build --force-reinstall --no-deps←[0m
Resolved 1 package in 4ms
   Built hello-pybind11 @ file:///D:/example
Prepared 1 package in 7.59s
Uninstalled 1 package in 1ms
Installed 1 package in 6ms
 ~ hello-pybind11==0.1.0 (from file:///D:/example)
⚡ Quick import test...
Hello, World from C++!
5 + 3 = 8

Execution time: 8.266 s

The current workflow feels unnecessarily slow for such a simple project—especially considering the source code hasn't even changed. Ideally, using CMake with Ninja would result in near-instant builds. I've attempted to streamline things by creating a couple of extra pre-dev and dev recipes, but they aren't working reliably. For example, when running tests, it's still using an outdated .pyd module.

My question is: what's the recommended, fastest way to achieve an optimal edit-compile-test cycle in a cross-platform manner? I'm using uv and just specifically to keep things portable, and I'm wondering if there are best practices or lesser-known features in scikit-build that could help here. Since I'm relatively new to it, I'd really appreciate guidance on how to set this up properly.

[LecrisUT]

The issue is probably because in isolated builds, pybind11 dependency is always moving so the cache is always invalidated. One option is to make the injection of pybind11 in the build-system.requires conditional. I don't think we have a way to do that, but would be an interesting feature and not too hard to do. And just to confirm this is the issue, can you try again with --no-build-isolation?

PS: There are various comments I would have on your CMakeLists.txt file, if you want to learn about some modern CMake design principles.

[brupelo]

@LecrisUT First of all, thanks for the feedback... here's what my next attempt looked like

Justfile:

# Global variables
pip_args := "--no-build-isolation --config-settings=build-dir=build"

default:
    @just --list

setup:
    @echo "🚀 Setting up development environment..."
    uv venv
    uv pip install build scikit-build-core pybind11
    @echo "✅ Initial development install..."
    uv pip install -e . {{pip_args}}
    @echo "🎉 Setup complete! Use 'just dev' for fast iterations."

pre-dev:
    @echo "⚡ Reconfiguring CMake with venv settings..."
    @uv run cmake -S . -B build \
        -Dpybind11_DIR=`uv run python -c "import pybind11; print(pybind11.get_cmake_dir())"` \
        -DPython_EXECUTABLE=`uv run python -c "import sys; print(sys.executable)"` \
        -DCMAKE_BUILD_TYPE=Release

dev:
    @echo "⚡ Building incrementally..."
    @uv run cmake --build build --config Release -j 4
    @echo "📦 Copying built module to site-packages..."
    @uv run python -c "import shutil, os, site; shutil.copy2(os.path.join('.venv', 'Release', 'hello_pybind11.cp311-win_amd64.pyd'), site.getsitepackages()[0])"

check:
    @echo "⚡ Quick import test..."
    @uv run python -c "import hello_pybind11; print(hello_pybind11.hello_world()); print(f'5 + 3 = {hello_pybind11.add(5, 3)}')"

test file="test.py":
    @echo "🧪 Running {{file}}..."
    @uv run python {{file}}

rebuild:
    @echo "🔄 Full rebuild..."
    @python -c "import shutil, os; [shutil.rmtree(p, ignore_errors=True) for p in ['build'] if os.path.exists(p)]"
    @python -c "import glob, os; [os.remove(f) for f in glob.glob('*.pyd') + glob.glob('*.so')]"
    @python -c "import shutil, glob; [shutil.rmtree(p, ignore_errors=True) for p in glob.glob('*.egg-info')]"
    uv pip install -e . {{pip_args}} --force-reinstall --no-deps

clean:
    @echo "🧹 Cleaning..."
    @python -c "import shutil, os; [shutil.rmtree(p, ignore_errors=True) for p in ['build', 'dist', '.venv'] if os.path.exists(p)]"
    @python -c "import glob, os; [os.remove(f) for f in glob.glob('*.pyd') + glob.glob('*.so')]"
    @python -c "import shutil, glob; [shutil.rmtree(p, ignore_errors=True) for p in glob.glob('*.egg-info')]"

info:
    @echo "📊 Build info:"
    @echo "pybind11 CMake Dir: `uv run python -c "import pybind11; print(pybind11.get_cmake_dir())"`"
    @echo "Python Executable: `uv run python -c "import sys; print(sys.executable)"`"
    @python -c "import os; print(f'Build dir: {os.path.abspath(\"build\")}')"
    @python -c "import sys; print(f'Python: {sys.executable}')"
    @python -c "import shutil; print(f'CMake: {\"found\" if shutil.which(\"cmake\") else \"not found\"}')"

profile:
    @echo "⏱️  Profiling build time..."
    @python -c "import time; start=time.time(); print(f'Starting build at: {time.strftime(\"%H:%M:%S\")}')"
    @python -c "import subprocess, time; start=time.time(); subprocess.run(['just', 'rebuild'], check=True, shell=True); print(f'Full rebuild time: {time.time()-start:.2f}s')"
    @python -c "import subprocess, time; start=time.time(); subprocess.run(['just', 'dev'], check=True, shell=True); print(f'Incremental dev time: {time.time()-start:.2f}s')"

main.cpp:

#include <pybind11/pybind11.h>
#include <string>

// Define the MACRO_STRINGIFY macro - handle empty values
#define STRINGIFY_HELPER(x) #x
#define STRINGIFY(x) STRINGIFY_HELPER(x)

std::string hello_world() {
    return "Hello, World 3 from C++!";
}

int add(int a, int b) {
    return a + b;
}

int sub(int a, int b) {
    return a - b;
}

namespace py = pybind11;

PYBIND11_MODULE(hello_pybind11, m) {
    m.doc() = "pybind11 hello world plugin";

    m.def("hello_world", &hello_world, "A function that returns a greeting");
    m.def("add", &add, "A function that adds two numbers");
    m.def("sub", &sub, "A function that subtracts two numbers");

    // Set version - handle empty VERSION_INFO gracefully
    #if defined(VERSION_INFO)
        m.attr("__version__") = STRINGIFY(VERSION_INFO);
    #else
        m.attr("__version__") = "dev";
    #endif
}

test.py:

from pathlib import Path

try:
    main_cpp_path = Path(__file__).parent / "main.cpp"
    print("Contents of main.cpp:\n")
    print(main_cpp_path.read_text())
except Exception as e:
    print(f"Failed to read main.cpp: {e}")

print("\n--- Running tests on hello_pybind11 module ---\n")

try:
    import hello_pybind11
    
    print("Testing hello_pybind11 module...")
    print(f"hello_world(): {hello_pybind11.hello_world()}")
    print(f"add(5, 3): {hello_pybind11.add(5, 3)}")
    print(f"sub(5, 3): {hello_pybind11.sub(5, 3)}")
    print(f"Version: {hello_pybind11.__version__}")
    print("All tests passed!")
    
except ImportError as e:
    print(f"Failed to import hello_pybind11: {e}")
    print("Make sure to build and install the module first")
except Exception as e:
    print(f"Error: {e}")

But this is strange — I tried following your earlier suggestion, but now I'm running
into a different issue that seems to be related to caching or stale files
caused somehow by uv and virtualenvs...

>ptime just setup clean test
🚀 Setting up development environment...
←[1muv venv←[0m
Using CPython 3.11.11
Creating virtual environment at: .venv
Activate with: .venv\Scripts\activate
uv pip install build scikit-build-core pybind11
Resolved 7 packages in 4ms
Installed 7 packages in 16ms
 + build==1.2.2.post1
 + colorama==0.4.6
 + packaging==25.0
 + pathspec==0.12.1
 + pybind11==3.0.0
 + pyproject-hooks==1.2.0
 + scikit-build-core==0.11.5
✅ Initial development install...
←[1muv pip install -e . --no-build-isolation --config-settings=build-dir=build←[0m
Resolved 1 package in 4ms
Installed 1 package in 4ms
 + hello-pybind11==0.1.0 (from file:///D:/mcve/test-scikit-build)
🎉 Setup complete! Use 'just dev' for fast iterations.
🧹 Cleaning...
🧪 Running test.py...
Using CPython 3.11.11
Creating virtual environment at: .venv
Installed 1 package in 4ms
Testing hello_pybind11 module...
hello_world(): Hello, World 2 from C++!
add(5, 3): 8
sub(5, 3): 2
Version: 0.1.0
All tests passed!

Execution time: 0.782 s`

You can see that for some reason, the latest changes in main.cpp aren't being
picked up, and even test.py isn't running the updated content.

That said, any advice on modern CMake practices or tooling would be very
welcome. What I'm aiming for is a robust, cross-platform development workflow
that works efficiently for larger projects involving C++ extensions in Python.
I want something that supports rapid iteration (edit/compile/test cycles),
while also being idempotent and deterministic enough for CI/CD pipelines —
without having to juggle various build tools or packaging quirks.

In short, I’d like to arrive at a canonical modern setup for building Python
bindings with C++ using scikit-build-core, without falling back to legacy pip
or setup.py tricks. The stack of just + uv + ninja should be sufficient — now
it’s about tightening the integration and making the workflow reliable and
repeatable across environments.

[LecrisUT]

You can see that for some reason, the latest changes in main.cpp aren't being picked up, and even test.py isn't running the updated content.

ccache and the likes are tools that I stay away from because it's not worth the issues it can cause. That said, can you try with sccache instead?

That said, any advice on modern CMake practices or tooling would be very
welcome.

If you can ping me on a repo, I'll write you a review there.

I want something that supports rapid iteration (edit/compile/test cycles)

There are a few improvements to be made, but editable install with redirect mode should be the development method you need. It would basically rebuild every time the import is run. Give that one a try and let us know of the issues you encounter. You should combine it with --no-build-issolation. I don't have experience with just, but this workflow should work for local development with your IDE.

[brupelo]

If you can ping me on a repo, I'll write you a review there.

That’s an excellent idea! To be honest pasting files in github issues is quite slow to iterate with. I’ve bootstrapped a small repo that you can find here.

Please don’t pay too much attention to the README.md as it’s just a draft and doesn’t fully reflect the current state, although the goal should be +/- clear. Hope it reflects well the current issue of this thread though.

---

ccache and the likes are tools that I stay away from because it's not worth the issues it can cause. That said, can you try with sccache instead?

I've referenced that one in the README, haven't tried yet, let's see if the issue with idempotency and repeateable builds is fixed first

There are a few improvements to be made, but editable install with redirect mode should be the development method you need. It would basically rebuild every time the import is run. Give that one a try and let us know of the issues you encounter. You should combine it with --no-build-issolation. I don't have experience with just, but this workflow should work for local development with your IDE.

I hope others with more experience in scikit-build can also help to this one, i've been using it for few hours only so I'm probably missing a lot of good practices.

[pboettch]

I was having the exact same question (as my build of the python lib is included in a global build and with isolation it was always rebuilt.

Using --no-build-isolation and a local build-dir, so "--no-build-isolation --config-settings=build-dir=build" added to pip install, solved my issue.