[eudsl][nbgen] eudsl-nbgen into eudsl-py working with cibuildwheel

.github/workflows/build_test_release_eudsl.yml

@@ -104,6 +104,19 @@ jobs:
           curl -sLO $RELEASE_URL
           tar xf $RELEASE_PREFIX*.tar.gz
           
+          if [[ "${{ matrix.os }}" == "ubuntu" ]]; then
+            echo "LLVM_DIR=/host/$PWD/llvm-install/lib/cmake/llvm" >> $GITHUB_ENV
+            echo "MLIR_DIR=/host/$PWD/llvm-install/lib/cmake/mlir" >> $GITHUB_ENV
+            echo "Clang_DIR=/host/$PWD/llvm-install/lib/cmake/clang" >> $GITHUB_ENV
+            echo "CCACHE_DIR=/host/$CCACHE_DIR" >> $GITHUB_ENV
+            echo "PIP_FIND_LINKS=/host/$PWD/wheelhouse" >> $GITHUB_ENV
+          else
+            echo "LLVM_DIR=$PWD/llvm-install/lib/cmake/llvm" >> $GITHUB_ENV
+            echo "MLIR_DIR=$PWD/llvm-install/lib/cmake/mlir" >> $GITHUB_ENV
+            echo "Clang_DIR=$PWD/llvm-install/lib/cmake/clang" >> $GITHUB_ENV
+            echo "PIP_FIND_LINKS=$PWD/wheelhouse" >> $GITHUB_ENV
+          fi
+          
           # since linux builds execute in the cibuildwheel almalinux container
           if [[ "${{ matrix.os }}" == "ubuntu" ]]; then
             echo CC=clang >> $GITHUB_ENV
@@ -123,35 +136,18 @@ jobs:
       - name: "Build eudsl-tblgen"
         run: |
           
-          if [[ "${{ matrix.os }}" == "ubuntu" ]]; then
-            export CMAKE_PREFIX_PATH="/host/$PWD/llvm-install"
-            export CCACHE_DIR="/host/$CCACHE_DIR"
-          else
-            export CMAKE_PREFIX_PATH="$PWD/llvm-install"
-          fi
-          
           $python3_command -m cibuildwheel "$PWD/projects/eudsl-tblgen" --output-dir wheelhouse
 
       - name: "Build eudsl-nbgen"
+        if: ${{ ! startsWith(matrix.os, 'windows') }}
         run: |
           
-          # this is not run in cibuildwheel container so its different
-          export CMAKE_PREFIX_PATH="$PWD/llvm-install"
-          $python3_command -m pip wheel "$PWD/projects/eudsl-nbgen" -w wheelhouse -v
+          $python3_command -m cibuildwheel "$PWD/projects/eudsl-nbgen" --output-dir wheelhouse
 
       - name: "Build eudsl-py"
         if: ${{ ! startsWith(matrix.os, 'windows') }}
         run: |
           
-          if [[ "${{ matrix.os }}" == "ubuntu" ]]; then
-            export CMAKE_PREFIX_PATH="/host/$PWD/llvm-install"
-            export CCACHE_DIR="/host/$CCACHE_DIR"
-            export PIP_FIND_LINKS="/host/$PWD/wheelhouse"
-          else
-            export CMAKE_PREFIX_PATH="$PWD/llvm-install"
-            export PIP_FIND_LINKS="$PWD/wheelhouse"
-          fi
-          
           # prevent OOM on free GHA
           export DISABLE_COMPILE_OPT="${{ inputs.debug_with_tmate }}"
           $python3_command -m cibuildwheel "$PWD/projects/eudsl-py" --output-dir wheelhouse
@@ -299,10 +295,22 @@ jobs:
       id-token: write
       contents: write
 
+    strategy:
+      fail-fast: false
+      matrix:
+        runs-on: ["ubuntu-22.04"]
+        include: [
+          {runs-on: "ubuntu-22.04", name: "ubuntu_x86_64"},
+          {runs-on: "ubuntu-22.04", name: "macos_arm64"},
+          {runs-on: "ubuntu-22.04", name: "macos_x86_64"},
+          {runs-on: "ubuntu-22.04", name: "windows_x86_64"},
+        ]
+
     steps:
 
       - uses: actions/download-artifact@v4
         with:
+          name: eudsl_${{ matrix.name }}_artifact
           path: wheelhouse
 
       - name: Release current commit

projects/eudsl-nbgen/CMakeLists.txt

@@ -87,6 +87,5 @@ install(
 install(
   FILES
   cmake/eudsl_nbgen-config.cmake
-  cmake/make_generated_shards.py
   DESTINATION "${EUDSL_NBGEN_INSTALL_DATADIR}/cmake"
 )

projects/eudsl-nbgen/cmake/eudsl_nbgen-config.cmake

@@ -5,23 +5,25 @@
 
 # copy-pasta from AddMLIR.cmake/AddLLVM.cmake/TableGen.cmake
 
-function(eudslpygen target inputFile)
-  set(EUDSLPYGEN_TARGET_DEFINITIONS ${inputFile})
+function(eudslpygen target input_file)
+  set(EUDSLPYGEN_TARGET_DEFINITIONS ${input_file})
   cmake_parse_arguments(ARG "" "" "DEPENDS;EXTRA_INCLUDES;NAMESPACES" ${ARGN})
   if (IS_ABSOLUTE ${EUDSLPYGEN_TARGET_DEFINITIONS})
-    set(EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE ${inputFile})
+    set(EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE ${input_file})
   else()
-    set(EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE ${CMAKE_CURRENT_SOURCE_DIR}/${inputFile})
+    set(EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE ${CMAKE_CURRENT_SOURCE_DIR}/${input_file})
   endif()
 
   if(NOT EXISTS "${EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE}")
-    message(FATAL_ERROR "${inputFile} does not exist")
+    message(FATAL_ERROR "${input_file} does not exist")
   endif()
 
   get_directory_property(eudslpygen_includes INCLUDE_DIRECTORIES)
-  list(APPEND eudslpygen_includes ${CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES} ${ARG_EXTRA_INCLUDES})
+  list(TRANSFORM ARG_EXTRA_INCLUDES PREPEND -I)
+  list(APPEND eudslpygen_includes ${CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES})
   list(REMOVE_ITEM eudslpygen_includes "")
   list(TRANSFORM eudslpygen_includes PREPEND -I)
+  list(APPEND eudslpygen_includes ${ARG_EXTRA_INCLUDES})
 
   set(_gen_target_dir "${CMAKE_CURRENT_BINARY_DIR}/generated/${target}")
   file(MAKE_DIRECTORY ${_gen_target_dir})
@@ -41,7 +43,7 @@ function(eudslpygen target inputFile)
     -o ${_depfile}
   )
   execute_process(COMMAND ${clang_command} RESULT_VARIABLE _had_error_depfile
-    COMMAND_ECHO STDOUT
+    # COMMAND_ECHO STDOUT
   )
 
   if (IS_ABSOLUTE ${EUDSLPYGEN_TARGET_DEFINITIONS})
@@ -64,19 +66,20 @@ function(eudslpygen target inputFile)
       -o "${_full_gen_file}"
     WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
     RESULT_VARIABLE _had_error_gen_cpp
-    COMMAND_ECHO STDOUT
+    # COMMAND_ECHO STDOUT
   )
   if((_had_error_gen_cpp AND NOT _had_error_gen_cpp EQUAL 0) OR NOT EXISTS "${_full_gen_file}")
     message(FATAL_ERROR "failed to create ${_full_gen_file}: ${_had_error_gen_cpp}")
   endif()
   # this is the specific thing connected the dependencies...
   set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${_full_gen_file})
   execute_process(
-    COMMAND ${Python_EXECUTABLE} ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/make_generated_shards.py
-      ${_full_gen_file} -t ${target} -I ${ARG_EXTRA_INCLUDES} ${EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE}
+    COMMAND ${EUDSLPY_EUDSLPYGEN_EXE} -shardify ${_full_gen_file}
+      # ARG_EXTRA_INCLUDES has already had -I prepended
+      -shard-target ${target} ${ARG_EXTRA_INCLUDES} -I ${EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE}
     WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
     RESULT_VARIABLE _had_error_gen_sharded
-    COMMAND_ECHO STDOUT
+    # COMMAND_ECHO STDOUT
   )
   if((_had_error_gen_sharded AND NOT _had_error_gen_sharded EQUAL 0) OR NOT EXISTS "${_full_gen_file}.sharded.cpp")
     message(FATAL_ERROR "failed to create ${_full_gen_file}.sharded.cpp: ${_had_error_gen_sharded}")
@@ -148,4 +151,4 @@ function(patch_mlir_llvm_rpath target)
     endif()
     set_target_properties(${target} PROPERTIES INSTALL_RPATH "${_origin_prefix}")
   endif()
-endfunction()
+endfunction()

projects/eudsl-nbgen/pyproject.toml

@@ -33,3 +33,55 @@ CMAKE_C_COMPILER_LAUNCHER = { env = "CMAKE_C_COMPILER_LAUNCHER", default = "" }
 CMAKE_CXX_COMPILER_LAUNCHER = { env = "CMAKE_CXX_COMPILER_LAUNCHER", default = "" }
 CMAKE_CXX_VISIBILITY_PRESET = "hidden"
 CMAKE_VERBOSE_MAKEFILE = "ON"
+
+[tool.cibuildwheel]
+build-verbosity = 1
+# build only one version
+build = ["cp39*"]
+skip = ["*-manylinux_i686", "*-musllinux*", "pp*", "*-win32"]
+archs = ["auto64"]
+manylinux-x86_64-image = "manylinux_2_28"
+environment-pass = [
+    "LLVM_DIR",
+    "CMAKE_GENERATOR",
+    "CMAKE_PREFIX_PATH",
+    "CC",
+    "CXX",
+    # ccache
+    "CCACHE_DIR",
+    "CCACHE_MAXSIZE=700M",
+    "CCACHE_SLOPPINESS",
+    "CCACHE_CPP2",
+    "CCACHE_UMASK",
+    "CMAKE_C_COMPILER_LAUNCHER",
+    "CMAKE_CXX_COMPILER_LAUNCHER"
+]
+before-build = [
+    "export CCACHE_DIR=$CCACHE_DIR/$(python -c 'import platform; print(platform.python_version())')",
+    "mkdir -p $CCACHE_DIR",
+    "ccache -z"
+]
+# uncomment to make sure ccache is working inside containers
+test-command = "ccache -s"
+repair-wheel-command = []
+
+[tool.cibuildwheel.linux]
+before-all = [
+    "yum install -y clang",
+    # ccache
+    "curl -sLO https://github.com/ccache/ccache/releases/download/v4.10.2/ccache-4.10.2-linux-x86_64.tar.xz",
+    "tar -xf ccache-4.10.2-linux-x86_64.tar.xz",
+    "pushd ccache-4.10.2-linux-x86_64 && make install && popd"
+]
+# synchronize TZ with host so ccache files have correct timestamp
+container-engine = { name = "docker", create-args = ["-v", "/etc/timezone:/etc/timezone:ro", "-v", "/etc/localtime:/etc/localtime:ro"] }
+
+[tool.cibuildwheel.macos]
+before-build = [
+    "ccache -z"
+]
+
+[tool.cibuildwheel.windows]
+before-build = [
+    "ccache -z"
+]