From ff3c39e28f3f7325ebf64452b9566272e513d689 Mon Sep 17 00:00:00 2001 From: Maksim Levental Date: Thu, 2 Jan 2025 18:30:46 -0500 Subject: [PATCH] [eudsl][nbgen] eudsl-nbgen into eudsl-py working with cibuildwheel --- .../workflows/build_test_release_eudsl.yml | 46 +++--- projects/eudsl-nbgen/CMakeLists.txt | 1 - .../cmake/eudsl_nbgen-config.cmake | 25 +-- .../cmake/make_generated_shards.py | 81 ---------- projects/eudsl-nbgen/pyproject.toml | 52 ++++++ projects/eudsl-nbgen/src/eudsl-nbgen.cpp | 151 +++++++++++++++++- projects/eudsl-py/src/eudslpy_ext.cpp | 3 +- 7 files changed, 241 insertions(+), 118 deletions(-) delete mode 100644 projects/eudsl-nbgen/cmake/make_generated_shards.py diff --git a/.github/workflows/build_test_release_eudsl.yml b/.github/workflows/build_test_release_eudsl.yml index e631e829..46d97d05 100644 --- a/.github/workflows/build_test_release_eudsl.yml +++ b/.github/workflows/build_test_release_eudsl.yml @@ -104,6 +104,19 @@ jobs: curl -sLO $RELEASE_URL tar xf $RELEASE_PREFIX*.tar.gz + if [[ "${{ matrix.os }}" == "ubuntu" ]]; then + echo "LLVM_DIR=/host/$PWD/llvm-install/lib/cmake/llvm" >> $GITHUB_ENV + echo "MLIR_DIR=/host/$PWD/llvm-install/lib/cmake/mlir" >> $GITHUB_ENV + echo "Clang_DIR=/host/$PWD/llvm-install/lib/cmake/clang" >> $GITHUB_ENV + echo "CCACHE_DIR=/host/$CCACHE_DIR" >> $GITHUB_ENV + echo "PIP_FIND_LINKS=/host/$PWD/wheelhouse" >> $GITHUB_ENV + else + echo "LLVM_DIR=$PWD/llvm-install/lib/cmake/llvm" >> $GITHUB_ENV + echo "MLIR_DIR=$PWD/llvm-install/lib/cmake/mlir" >> $GITHUB_ENV + echo "Clang_DIR=$PWD/llvm-install/lib/cmake/clang" >> $GITHUB_ENV + echo "PIP_FIND_LINKS=$PWD/wheelhouse" >> $GITHUB_ENV + fi + # since linux builds execute in the cibuildwheel almalinux container if [[ "${{ matrix.os }}" == "ubuntu" ]]; then echo CC=clang >> $GITHUB_ENV @@ -123,35 +136,18 @@ jobs: - name: "Build eudsl-tblgen" run: | - if [[ "${{ matrix.os }}" == "ubuntu" ]]; then - export CMAKE_PREFIX_PATH="/host/$PWD/llvm-install" - export CCACHE_DIR="/host/$CCACHE_DIR" - else - export CMAKE_PREFIX_PATH="$PWD/llvm-install" - fi - $python3_command -m cibuildwheel "$PWD/projects/eudsl-tblgen" --output-dir wheelhouse - name: "Build eudsl-nbgen" + if: ${{ ! startsWith(matrix.os, 'windows') }} run: | - # this is not run in cibuildwheel container so its different - export CMAKE_PREFIX_PATH="$PWD/llvm-install" - $python3_command -m pip wheel "$PWD/projects/eudsl-nbgen" -w wheelhouse -v + $python3_command -m cibuildwheel "$PWD/projects/eudsl-nbgen" --output-dir wheelhouse - name: "Build eudsl-py" if: ${{ ! startsWith(matrix.os, 'windows') }} run: | - if [[ "${{ matrix.os }}" == "ubuntu" ]]; then - export CMAKE_PREFIX_PATH="/host/$PWD/llvm-install" - export CCACHE_DIR="/host/$CCACHE_DIR" - export PIP_FIND_LINKS="/host/$PWD/wheelhouse" - else - export CMAKE_PREFIX_PATH="$PWD/llvm-install" - export PIP_FIND_LINKS="$PWD/wheelhouse" - fi - # prevent OOM on free GHA export DISABLE_COMPILE_OPT="${{ inputs.debug_with_tmate }}" $python3_command -m cibuildwheel "$PWD/projects/eudsl-py" --output-dir wheelhouse @@ -299,10 +295,22 @@ jobs: id-token: write contents: write + strategy: + fail-fast: false + matrix: + runs-on: ["ubuntu-22.04"] + include: [ + {runs-on: "ubuntu-22.04", name: "ubuntu_x86_64"}, + {runs-on: "ubuntu-22.04", name: "macos_arm64"}, + {runs-on: "ubuntu-22.04", name: "macos_x86_64"}, + {runs-on: "ubuntu-22.04", name: "windows_x86_64"}, + ] + steps: - uses: actions/download-artifact@v4 with: + name: eudsl_${{ matrix.name }}_artifact path: wheelhouse - name: Release current commit diff --git a/projects/eudsl-nbgen/CMakeLists.txt b/projects/eudsl-nbgen/CMakeLists.txt index b1e92720..b5faa3f3 100644 --- a/projects/eudsl-nbgen/CMakeLists.txt +++ b/projects/eudsl-nbgen/CMakeLists.txt @@ -87,6 +87,5 @@ install( install( FILES cmake/eudsl_nbgen-config.cmake - cmake/make_generated_shards.py DESTINATION "${EUDSL_NBGEN_INSTALL_DATADIR}/cmake" ) diff --git a/projects/eudsl-nbgen/cmake/eudsl_nbgen-config.cmake b/projects/eudsl-nbgen/cmake/eudsl_nbgen-config.cmake index bfdc4968..09c0ede8 100644 --- a/projects/eudsl-nbgen/cmake/eudsl_nbgen-config.cmake +++ b/projects/eudsl-nbgen/cmake/eudsl_nbgen-config.cmake @@ -5,23 +5,25 @@ # copy-pasta from AddMLIR.cmake/AddLLVM.cmake/TableGen.cmake -function(eudslpygen target inputFile) - set(EUDSLPYGEN_TARGET_DEFINITIONS ${inputFile}) +function(eudslpygen target input_file) + set(EUDSLPYGEN_TARGET_DEFINITIONS ${input_file}) cmake_parse_arguments(ARG "" "" "DEPENDS;EXTRA_INCLUDES;NAMESPACES" ${ARGN}) if (IS_ABSOLUTE ${EUDSLPYGEN_TARGET_DEFINITIONS}) - set(EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE ${inputFile}) + set(EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE ${input_file}) else() - set(EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE ${CMAKE_CURRENT_SOURCE_DIR}/${inputFile}) + set(EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE ${CMAKE_CURRENT_SOURCE_DIR}/${input_file}) endif() if(NOT EXISTS "${EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE}") - message(FATAL_ERROR "${inputFile} does not exist") + message(FATAL_ERROR "${input_file} does not exist") endif() get_directory_property(eudslpygen_includes INCLUDE_DIRECTORIES) - list(APPEND eudslpygen_includes ${CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES} ${ARG_EXTRA_INCLUDES}) + list(TRANSFORM ARG_EXTRA_INCLUDES PREPEND -I) + list(APPEND eudslpygen_includes ${CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES}) list(REMOVE_ITEM eudslpygen_includes "") list(TRANSFORM eudslpygen_includes PREPEND -I) + list(APPEND eudslpygen_includes ${ARG_EXTRA_INCLUDES}) set(_gen_target_dir "${CMAKE_CURRENT_BINARY_DIR}/generated/${target}") file(MAKE_DIRECTORY ${_gen_target_dir}) @@ -41,7 +43,7 @@ function(eudslpygen target inputFile) -o ${_depfile} ) execute_process(COMMAND ${clang_command} RESULT_VARIABLE _had_error_depfile - COMMAND_ECHO STDOUT + # COMMAND_ECHO STDOUT ) if (IS_ABSOLUTE ${EUDSLPYGEN_TARGET_DEFINITIONS}) @@ -64,7 +66,7 @@ function(eudslpygen target inputFile) -o "${_full_gen_file}" WORKING_DIRECTORY ${CMAKE_BINARY_DIR} RESULT_VARIABLE _had_error_gen_cpp - COMMAND_ECHO STDOUT + # COMMAND_ECHO STDOUT ) if((_had_error_gen_cpp AND NOT _had_error_gen_cpp EQUAL 0) OR NOT EXISTS "${_full_gen_file}") message(FATAL_ERROR "failed to create ${_full_gen_file}: ${_had_error_gen_cpp}") @@ -72,11 +74,12 @@ function(eudslpygen target inputFile) # this is the specific thing connected the dependencies... set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${_full_gen_file}) execute_process( - COMMAND ${Python_EXECUTABLE} ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/make_generated_shards.py - ${_full_gen_file} -t ${target} -I ${ARG_EXTRA_INCLUDES} ${EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE} + COMMAND ${EUDSLPY_EUDSLPYGEN_EXE} -shardify ${_full_gen_file} + # ARG_EXTRA_INCLUDES has already had -I prepended + -shard-target ${target} ${ARG_EXTRA_INCLUDES} -I ${EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR} RESULT_VARIABLE _had_error_gen_sharded - COMMAND_ECHO STDOUT + # COMMAND_ECHO STDOUT ) if((_had_error_gen_sharded AND NOT _had_error_gen_sharded EQUAL 0) OR NOT EXISTS "${_full_gen_file}.sharded.cpp") message(FATAL_ERROR "failed to create ${_full_gen_file}.sharded.cpp: ${_had_error_gen_sharded}") diff --git a/projects/eudsl-nbgen/cmake/make_generated_shards.py b/projects/eudsl-nbgen/cmake/make_generated_shards.py deleted file mode 100644 index 207fe4f4..00000000 --- a/projects/eudsl-nbgen/cmake/make_generated_shards.py +++ /dev/null @@ -1,81 +0,0 @@ -# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# Copyright (c) 2024. - -import argparse -import re -from pathlib import Path -from textwrap import dedent - - -def make_source_shards(filename: Path, target, extra_includes, max_num_shards): - assert filename.name.endswith("cpp.gen"), "expected .cpp.gen file" - with open(filename) as f: - source = f.read() - shards = re.split(r"// eudslpy-gen-shard \d+", source) - if target is None: - target = filename.stem.split(".")[0] - for i, shar in enumerate(shards): - with open(f"{filename}.shard.{i}.cpp", "w") as f: - if extra_includes is not None: - for inc in extra_includes: - print(f'#include "{inc}"', file=f) - print( - dedent( - f"""\ - #include - namespace nb = nanobind; - using namespace nb::literals; - using namespace mlir; - using namespace llvm; - #include "type_casters.h" - - void populate{target}{i}Module(nb::module_ &m) {{""" - ), - file=f, - ) - print(shar, file=f) - print("}", file=f) - - if max_num_shards is not None: - if len(shards) > max_num_shards: - raise RuntimeError(f"expected less than {max_num_shards} shards") - for i in range(len(shards), max_num_shards): - with open(f"{filename}.shard.{i}.cpp", "w") as f: - print(f"// dummy shard {i}", file=f) - - with open(f"{filename}.sharded.cpp", "w") as f: - print( - dedent( - f"""\ - #include - namespace nb = nanobind; - using namespace nb::literals; - void populate{target}Module(nb::module_ &m) {{""" - ), - file=f, - ) - for i in range(len(shards)): - print( - dedent(f"extern void populate{target}{i}Module(nb::module_ &m);"), - file=f, - ) - for i in range(len(shards)): - print(dedent(f"populate{target}{i}Module(m);"), file=f) - - print("}", file=f) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("filename") - parser.add_argument("-t", "--target") - parser.add_argument("-I", "--extra_includes", nargs="*") - parser.add_argument("-m", "--max-num-shards", type=int) - args = parser.parse_args() - if args.max_num_shards: - args.max_num_shards += 1 - make_source_shards( - Path(args.filename), args.target, args.extra_includes, args.max_num_shards - ) diff --git a/projects/eudsl-nbgen/pyproject.toml b/projects/eudsl-nbgen/pyproject.toml index f1e58aae..d6617a6b 100644 --- a/projects/eudsl-nbgen/pyproject.toml +++ b/projects/eudsl-nbgen/pyproject.toml @@ -33,3 +33,55 @@ CMAKE_C_COMPILER_LAUNCHER = { env = "CMAKE_C_COMPILER_LAUNCHER", default = "" } CMAKE_CXX_COMPILER_LAUNCHER = { env = "CMAKE_CXX_COMPILER_LAUNCHER", default = "" } CMAKE_CXX_VISIBILITY_PRESET = "hidden" CMAKE_VERBOSE_MAKEFILE = "ON" + +[tool.cibuildwheel] +build-verbosity = 1 +# build only one version +build = ["cp39*"] +skip = ["*-manylinux_i686", "*-musllinux*", "pp*", "*-win32"] +archs = ["auto64"] +manylinux-x86_64-image = "manylinux_2_28" +environment-pass = [ + "LLVM_DIR", + "CMAKE_GENERATOR", + "CMAKE_PREFIX_PATH", + "CC", + "CXX", + # ccache + "CCACHE_DIR", + "CCACHE_MAXSIZE=700M", + "CCACHE_SLOPPINESS", + "CCACHE_CPP2", + "CCACHE_UMASK", + "CMAKE_C_COMPILER_LAUNCHER", + "CMAKE_CXX_COMPILER_LAUNCHER" +] +before-build = [ + "export CCACHE_DIR=$CCACHE_DIR/$(python -c 'import platform; print(platform.python_version())')", + "mkdir -p $CCACHE_DIR", + "ccache -z" +] +# uncomment to make sure ccache is working inside containers +test-command = "ccache -s" +repair-wheel-command = [] + +[tool.cibuildwheel.linux] +before-all = [ + "yum install -y clang", + # ccache + "curl -sLO https://github.com/ccache/ccache/releases/download/v4.10.2/ccache-4.10.2-linux-x86_64.tar.xz", + "tar -xf ccache-4.10.2-linux-x86_64.tar.xz", + "pushd ccache-4.10.2-linux-x86_64 && make install && popd" +] +# synchronize TZ with host so ccache files have correct timestamp +container-engine = { name = "docker", create-args = ["-v", "/etc/timezone:/etc/timezone:ro", "-v", "/etc/localtime:/etc/localtime:ro"] } + +[tool.cibuildwheel.macos] +before-build = [ + "ccache -z" +] + +[tool.cibuildwheel.windows] +before-build = [ + "ccache -z" +] diff --git a/projects/eudsl-nbgen/src/eudsl-nbgen.cpp b/projects/eudsl-nbgen/src/eudsl-nbgen.cpp index 0b65ab64..6db162f1 100644 --- a/projects/eudsl-nbgen/src/eudsl-nbgen.cpp +++ b/projects/eudsl-nbgen/src/eudsl-nbgen.cpp @@ -15,7 +15,9 @@ #include "llvm/Support/FormatVariadic.h" #include "llvm/Support/ToolOutputFile.h" +#include #include +#include static llvm::cl::OptionCategory EUDSLPYGenCat("Options for eudslpy-gen"); static llvm::cl::opt InputFilename(llvm::cl::Positional, @@ -30,7 +32,7 @@ static llvm::cl::list static llvm::cl::opt OutputFilename("o", llvm::cl::desc("Output filename"), - llvm::cl::value_desc("filename"), llvm::cl::Required, + llvm::cl::value_desc("filename"), llvm::cl::cat(EUDSLPYGenCat)); static llvm::cl::list @@ -47,6 +49,22 @@ static llvm::cl::opt ShardSize("shard-size", llvm::cl::desc("Shard size"), llvm::cl::cat(EUDSLPYGenCat), llvm::cl::init(10)); +static llvm::cl::opt Shardify("shardify", + llvm::cl::desc("Split into shards"), + llvm::cl::value_desc("shardify"), + llvm::cl::cat(EUDSLPYGenCat), + llvm::cl::init(false)); + +static llvm::cl::opt + MaxNumShards("max-number-shards", + llvm::cl::desc("Maximum number of shards to split into"), + llvm::cl::value_desc("max number shards"), + llvm::cl::cat(EUDSLPYGenCat), llvm::cl::init(-1)); + +static llvm::cl::opt ShardTarget( + "shard-target", llvm::cl::desc("CMake target that is being sharded"), + llvm::cl::value_desc("sharding target"), llvm::cl::cat(EUDSLPYGenCat)); + static bool filterInNamespace(const std::string &s) { if (Namespaces.empty()) return true; @@ -648,9 +666,7 @@ struct GenerateBindingsAction : clang::ASTFrontendAction { std::shared_ptr outputFile; }; -int main(int argc, char **argv) { - llvm::cl::ParseCommandLineOptions(argc, argv); - +static int generate() { llvm::ErrorOr> fileOrErr = llvm::MemoryBuffer::getFileOrSTDIN(InputFilename); if (std::error_code ec = fileOrErr.getError()) { @@ -720,3 +736,130 @@ int main(int argc, char **argv) { return 0; } + +static int makeSourceShards() { + // Ensure the filename ends with ".cpp.gen" + if (InputFilename.substr(InputFilename.size() - 8) != ".cpp.gen") { + llvm::errs() << "expected .cpp.gen file\n"; + return -1; + } + + std::ifstream file(InputFilename); + if (!file.is_open()) { + llvm::errs() << "Failed to open file\n"; + return -1; + } + + std::stringstream buffer; + buffer << file.rdbuf(); + std::string source = buffer.str(); + + // Split source by regex pattern + std::regex re("// eudslpy-gen-shard \\d+"); + std::sregex_token_iterator iter(source.begin(), source.end(), re, -1); + std::sregex_token_iterator end; + + std::vector shards; + while (iter != end) + shards.push_back(*iter++); + + std::string finalTarget = + ShardTarget.empty() + ? InputFilename.substr(0, InputFilename.find_last_of(".")) + : ShardTarget; + + // Generate shard files + + for (size_t i = 0; i < shards.size(); ++i) { + std::ofstream shardFile(InputFilename.getValue() + ".shard." + + std::to_string(i) + ".cpp"); + if (!shardFile.is_open()) { + llvm::errs() << "Failed to create shard file"; + return -1; + } + + if (!IncludeDirs.empty()) { + for (const std::string &inc : IncludeDirs) + shardFile << "#include \"" << inc << "\"" << std::endl; + } + + // clang-format off + shardFile << R"( +#include +namespace nb = nanobind; +using namespace nb::literals; +using namespace mlir; +using namespace llvm; +#include "type_casters.h" + +void populate)" << finalTarget << i << R"(Module(nb::module_ &m) { +)"; + // clang-format on + shardFile << shards[i] << std::endl; + shardFile << "}" << std::endl; + shardFile.flush(); + shardFile.close(); + } + + // Handle max number of shards + if (MaxNumShards != -1 && shards.size() > static_cast(MaxNumShards)) { + llvm::errs() << "expected less than " + std::to_string(MaxNumShards) + + " shards"; + return -1; + } + if (MaxNumShards == -1) + MaxNumShards = shards.size(); + + for (size_t i = shards.size(); i < static_cast(MaxNumShards); ++i) { + std::ofstream dummyShardFile(InputFilename.getValue() + ".shard." + + std::to_string(i) + ".cpp"); + if (!dummyShardFile.is_open()) { + llvm::errs() << "Failed to create dummy shard file"; + return -1; + } + dummyShardFile << "// dummy shard " << i << std::endl; + dummyShardFile.flush(); + dummyShardFile.close(); + } + + // Generate the final sharded cpp file + std::ofstream finalShardedFile(InputFilename.getValue() + ".sharded.cpp"); + if (!finalShardedFile.is_open()) { + llvm::errs() << "Failed to create final sharded file"; + return -1; + } + + // clang-format off + finalShardedFile << R"( +#include +namespace nb = nanobind; +using namespace nb::literals; + +void populate)" << finalTarget << R"(Module(nb::module_ &m) { +)"; + // clang-format on + + for (size_t i = 0; i < shards.size(); ++i) + finalShardedFile << "extern void populate" << finalTarget << i + << "Module(nb::module_ &m);" << std::endl; + + for (size_t i = 0; i < shards.size(); ++i) + finalShardedFile << "populate" << finalTarget << i << "Module(m);" + << std::endl; + + finalShardedFile << "}" << std::endl; + finalShardedFile.flush(); + finalShardedFile.close(); + + return 0; +} + +int main(int argc, char **argv) { + llvm::cl::ParseCommandLineOptions(argc, argv); + + if (Shardify) + return makeSourceShards(); + return generate(); + + return 0; +} diff --git a/projects/eudsl-py/src/eudslpy_ext.cpp b/projects/eudsl-py/src/eudslpy_ext.cpp index b527527a..0b3fc7dc 100644 --- a/projects/eudsl-py/src/eudslpy_ext.cpp +++ b/projects/eudsl-py/src/eudslpy_ext.cpp @@ -1,8 +1,7 @@ -// Copyright (c) 2025. -// // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// Copyright (c) 2024. #include #include