diff --git a/.github/workflows/build_test_release_eudsl.yml b/.github/workflows/build_test_release_eudsl.yml index ce31af07..bb2a3f1f 100644 --- a/.github/workflows/build_test_release_eudsl.yml +++ b/.github/workflows/build_test_release_eudsl.yml @@ -108,7 +108,6 @@ jobs: echo "LLVM_DIR=/host/$PWD/llvm-install/lib/cmake/llvm" >> $GITHUB_ENV echo "MLIR_DIR=/host/$PWD/llvm-install/lib/cmake/mlir" >> $GITHUB_ENV echo "Clang_DIR=/host/$PWD/llvm-install/lib/cmake/clang" >> $GITHUB_ENV - echo "CCACHE_DIR=/host/$CCACHE_DIR" >> $GITHUB_ENV echo "PIP_FIND_LINKS=/host/$PWD/wheelhouse" >> $GITHUB_ENV else echo "LLVM_DIR=$PWD/llvm-install/lib/cmake/llvm" >> $GITHUB_ENV @@ -131,27 +130,47 @@ jobs: echo MACOSX_DEPLOYMENT_TARGET=13.7 >> $GITHUB_ENV fi + # prevent OOM on free GHA + echo "EUDSLPY_DISABLE_COMPILE_OPT=${{ inputs.eudslpy_disable_compile_opt == 'true' && 'ON' || 'OFF' }}" >> $GITHUB_ENV + pip install cibuildwheel - name: "Build eudsl-tblgen" run: | + if [[ "${{ matrix.os }}" == "ubuntu" ]]; then + export CCACHE_DIR=/host/$CCACHE_DIR + fi $python3_command -m cibuildwheel "$PWD/projects/eudsl-tblgen" --output-dir wheelhouse - name: "Build eudsl-nbgen" - if: ${{ ! startsWith(matrix.os, 'windows') }} run: | + if [[ "${{ matrix.os }}" == "ubuntu" ]]; then + export CCACHE_DIR=/host/$CCACHE_DIR + fi $python3_command -m cibuildwheel "$PWD/projects/eudsl-nbgen" --output-dir wheelhouse - name: "Build eudsl-py" if: ${{ ! startsWith(matrix.os, 'windows') }} run: | - # prevent OOM on free GHA - export EUDSLPY_DISABLE_COMPILE_OPT="${{ inputs.eudslpy_disable_compile_opt == 'true' && 'ON' || 'OFF' }}" + if [[ "${{ matrix.os }}" == "ubuntu" ]]; then + export CCACHE_DIR=/host/$CCACHE_DIR + fi $python3_command -m cibuildwheel "$PWD/projects/eudsl-py" --output-dir wheelhouse + # just to/make sure total build continues to work + - name: "Build all of eudsl" + run: | + + pip install -r requirements.txt + cmake -B $PWD/eudsl-build -S $PWD \ + -DCMAKE_PREFIX_PATH=$PWD/llvm-install \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$PWD/eudsl-install + cmake --build "$PWD/eudsl-build" --target install + - name: "Save cache" uses: actions/cache/save@v3 if: ${{ !cancelled() && github.event_name == 'push' && github.ref_name == 'main' }} diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000..4f423581 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,75 @@ +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# Copyright (c) 2024. + +cmake_minimum_required(VERSION 3.29) +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +set(LLVM_SUBPROJECT_TITLE "EUDSL") +set(EUDSL_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}) + +if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + message("Building ${LLVM_SUBPROJECT_TITLE} as a standalone project.") + project(${LLVM_SUBPROJECT_TITLE} CXX C) + set(EUDSL_STANDALONE_BUILD ON) +else() + enable_language(CXX C) + set(EUDSL_STANDALONE_BUILD OFF) +endif() + +if(EUDSL_STANDALONE_BUILD) + find_package(LLVM REQUIRED CONFIG) + find_package(MLIR REQUIRED CONFIG) + find_package(Clang REQUIRED CONFIG PATHS "${LLVM_BINARY_DIR}" NO_DEFAULT_PATH NO_CMAKE_FIND_ROOT_PATH) + + message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") + message(STATUS "Using MLIRConfig.cmake in: ${MLIR_DIR}") + message(STATUS "Using ClangConfig.cmake in: ${Clang_DIR}") + + set(LLVM_RUNTIME_OUTPUT_INTDIR ${CMAKE_BINARY_DIR}/bin) + set(LLVM_LIBRARY_OUTPUT_INTDIR ${CMAKE_BINARY_DIR}/lib) + set(MLIR_BINARY_DIR ${CMAKE_BINARY_DIR}) + + list(APPEND CMAKE_MODULE_PATH "${MLIR_CMAKE_DIR}") + list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_DIR}") + list(APPEND CMAKE_MODULE_PATH "${CLANG_CMAKE_DIR}") + + include(TableGen) + include(AddLLVM) + include(AddMLIR) + include(AddClang) + + set(MLIR_INCLUDE_DIR ${MLIR_INCLUDE_DIRS}) +else() + # turning LLVM -DLLVM_OPTIMIZED_TABLEGEN=ON builds some stuff in the NATIVE dir + # but not everything so LLVM_BINARY_DIR isn't correct + string(REPLACE "NATIVE" "" LLVM_BINARY_DIR "${LLVM_BINARY_DIR}") + # Build via external projects mechanism + set(LLVM_INCLUDE_DIR ${LLVM_MAIN_SRC_DIR}/include) + set(LLVM_GENERATED_INCLUDE_DIR ${LLVM_BINARY_DIR}/include) + set(LLVM_INCLUDE_DIRS "${LLVM_INCLUDE_DIR};${LLVM_GENERATED_INCLUDE_DIR}") + + set(MLIR_MAIN_SRC_DIR ${LLVM_MAIN_SRC_DIR}/../mlir) + set(MLIR_INCLUDE_DIR ${MLIR_MAIN_SRC_DIR}/include) + set(MLIR_GENERATED_INCLUDE_DIR ${LLVM_BINARY_DIR}/tools/mlir/include) + set(MLIR_INCLUDE_DIRS "${MLIR_INCLUDE_DIR};${MLIR_GENERATED_INCLUDE_DIR}") + + set(CLANG_MAIN_SRC_DIR ${LLVM_MAIN_SRC_DIR}/../clang) + set(CLANG_INCLUDE_DIR ${CLANG_MAIN_SRC_DIR}/include) + set(CLANG_GENERATED_INCLUDE_DIR ${LLVM_BINARY_DIR}/tools/clang/include) + set(CLANG_INCLUDE_DIRS "${CLANG_INCLUDE_DIR};${CLANG_GENERATED_INCLUDE_DIR}") +endif() + +include_directories(${LLVM_INCLUDE_DIRS}) +include_directories(${MLIR_INCLUDE_DIRS}) +include_directories(${CLANG_INCLUDE_DIRS}) +link_directories(${LLVM_BUILD_LIBRARY_DIR}) +add_definitions(${LLVM_DEFINITIONS}) + +if(NOT TARGET LLVMSupport) + message(FATAL_ERROR "LLVMSupport not found") +endif() + +add_subdirectory(projects) \ No newline at end of file diff --git a/README.md b/README.md index 1208cabe..773d3c51 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,68 @@ This repository contains the source code for `EUDSL`, a toolkit for the construction of embedded DSLs, in arbitrary languages, for targeting [MLIR](https://mlir.llvm.org). +FYI: this project is currently "alpha" quality. + +Currently there are three components: + +1. [eudsl-tblgen](./projects/eudsl-tblgen): Python bindings to [MLIR's TableGen library](https://github.com/llvm/llvm-project/tree/main/mlir/lib/TableGen); +2. [eudsl-nbgen](./projects/eudsl-nbgen): A source-to-source translator that translates MLIR headers[^1] into direct `nanobind` bindings; +3. [eudsl-py](./projects/eudsl-py): Direct Python bindings to MLIR, generated using `eudsl-nbgen`; + * Currently only TableGen outputs (various `*.h.inc` files generated by `mlir-tblgen`) are automatically translated but the `eudsl-nbgen` tool can be manually run to translate any MLIR header (to varying degrees of success); + * See [projects/eudsl-py/tests](./projects/eudsl-py/tests) for a few examples of what these bindings look like. + ## Getting started -Coming soon... +Python wheels of all the tools are available at the [`latest` release page](https://github.com/llvm/eudsl/releases/tag/latest). +They are also `pip install`-able with .e.g + +```shell +$ cd +$ pip install eudsl-py -f https://github.com/llvm/eudsl/releases/expanded_assets/latest +``` + +`eudsl-py` has a slowly growing set of tests @ [projects/eudsl-py/tests](./projects/eudsl-py/tests). + +If you don't want to install locally, here is a [colab notebook minimal working example](https://colab.research.google.com/drive/1l-6rVnsUM3ypn7rKcaF_V6XVdopEM4Df?usp=sharing). + +## Developing/building + +**Strong recommendation**: check the CI scripts @ [.github/workflows](.github/workflows) - they do a fresh checkout and build on every commit and are written to be read by a non-CI expert. + +Firstly, you need a distribution of LLVM. You can build LLVM from source using our submodule by doing (on Mac/Linux or mingw): + +```shell +$ cd +$ ./build_tools/build_llvm.sh +``` + +Alternatively you can download a distribution for your platform from the [`llvm` release page](https://github.com/llvm/eudsl/releases/tag/llvm). + +Then each of the above tools can both be built as a conventional, standalone, CMake project and as a Python wheel. +The wheel build looks something like: + +```shell +$ cd +$ export CMAKE_PREFIX_PATH=$PWD/llvm-install +$ pip wheel projects/eudsl-nbgen -w wheelhouse -v +$ pip wheel projects/eudsl-py -w wheelhouse -v --find-links $PWD/wheelhouse +``` + +Note, the trailing `--find-links $PWD/wheelhouse` on `pip wheel projects/eudsl-py` is because `eudsl-nbgen` is a dependency of `eudsl-py` (that can be satisfied using the `eudsl-nbgen` wheel). + +If you want to build an individual tool via CMake you can do something like: + +```shell +$ cd +$ pip install -r requirements.txt +$ export CMAKE_PREFIX_PATH=$PWD/llvm-install +$ cmake -B $PWD/eudsl-nbgen-build -S $PWD/projects/eudsl-nbgen -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$PWD/eudsl-nbgen-install +$ cmake --build "$PWD/eudsl-build" --target install +``` + +If you want to build all the tools at once using CMake you can use the root [`CMakeLists.txt`](./CMakeLists.txt). +Note, in this case, `eudsl-nbgen` will automatically be built prior to `eudsl-py`. + +## Footnotes + +[^1]: Yes C++ headers... \ No newline at end of file diff --git a/projects/CMakeLists.txt b/projects/CMakeLists.txt new file mode 100644 index 00000000..e3b16175 --- /dev/null +++ b/projects/CMakeLists.txt @@ -0,0 +1,10 @@ +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# Copyright (c) 2024. + +if(NOT WIN32) + add_subdirectory(eudsl-py) +endif() +add_subdirectory(eudsl-nbgen) +add_subdirectory(eudsl-tblgen) diff --git a/projects/eudsl-nbgen/CMakeLists.txt b/projects/eudsl-nbgen/CMakeLists.txt index b5faa3f3..a63b9287 100644 --- a/projects/eudsl-nbgen/CMakeLists.txt +++ b/projects/eudsl-nbgen/CMakeLists.txt @@ -30,25 +30,7 @@ if(EUDSL_NBGEN_STANDALONE_BUILD) include(TableGen) include(AddLLVM) include(AddClang) - # TODO(max): probably don't need this anymore after landing the nanobind fix? - # technically we need this on windows too but our LLVM is compiled without exception handling - # and that breaks windows - if(NOT WIN32) - include(HandleLLVMOptions) - endif() -else() - # turning LLVM -DLLVM_OPTIMIZED_TABLEGEN=ON builds some stuff in the NATIVE dir - # but not everything so LLVM_BINARY_DIR isn't correct - string(REPLACE "NATIVE" "" LLVM_BINARY_DIR "${LLVM_BINARY_DIR}") - # Build via external projects mechanism - set(LLVM_INCLUDE_DIR ${LLVM_MAIN_SRC_DIR}/include) - set(LLVM_GENERATED_INCLUDE_DIR ${LLVM_BINARY_DIR}/include) - set(LLVM_INCLUDE_DIRS "${LLVM_INCLUDE_DIR};${LLVM_GENERATED_INCLUDE_DIR}") - - set(CLANG_MAIN_SRC_DIR ${LLVM_MAIN_SRC_DIR}/../clang) - set(CLANG_INCLUDE_DIR ${CLANG_MAIN_SRC_DIR}/include) - set(CLANG_GENERATED_INCLUDE_DIR ${LLVM_BINARY_DIR}/tools/clang/include) - set(CLANG_INCLUDE_DIRS "${CLANG_INCLUDE_DIR};${CLANG_GENERATED_INCLUDE_DIR}") + include(HandleLLVMOptions) endif() include_directories(${LLVM_INCLUDE_DIRS}) @@ -72,13 +54,25 @@ target_link_libraries(eudsl-nbgen ) string(TOLOWER ${LLVM_SUBPROJECT_TITLE} EUDSL_NBGEN_INSTALL_DATADIR) -# https://github.com/scikit-build/scikit-build-core/blob/a887a9b6c057b4ce9d3cfd53ae24e73caf1395a2/docs/build.md?plain=1#L139-L148 -# actually installs to venv/bin -install(TARGETS eudsl-nbgen RUNTIME DESTINATION "${EUDSL_NBGEN_INSTALL_DATADIR}.data/scripts") -# this actually installs into venv/lib -install(IMPORTED_RUNTIME_ARTIFACTS LLVM LIBRARY DESTINATION "${EUDSL_NBGEN_INSTALL_DATADIR}.data/data/lib") +if (NOT "$ENV{PIP_BUILD_TRACKER}" STREQUAL "") + # pip install + # actually installs to venv/bin + # https://github.com/scikit-build/scikit-build-core/blob/a887a9b6c057b4ce9d3cfd53ae24e73caf1395a2/docs/build.md?plain=1#L139-L148 + install(TARGETS eudsl-nbgen RUNTIME DESTINATION "${EUDSL_NBGEN_INSTALL_DATADIR}.data/scripts") + if (NOT WIN32) + # this actually installs into venv/lib + install(IMPORTED_RUNTIME_ARTIFACTS LLVM LIBRARY DESTINATION "${EUDSL_NBGEN_INSTALL_DATADIR}.data/data/lib") + endif() +else() + # pip cmake install + install(TARGETS eudsl-nbgen RUNTIME DESTINATION "${EUDSL_NBGEN_INSTALL_DATADIR}/bin") + if (NOT WIN32) + install(IMPORTED_RUNTIME_ARTIFACTS LLVM LIBRARY DESTINATION "${EUDSL_NBGEN_INSTALL_DATADIR}/lib") + endif() +endif() install( + # the slash is load-bearing... DIRECTORY src/ DESTINATION "${EUDSL_NBGEN_INSTALL_DATADIR}" FILES_MATCHING PATTERN "*\.py" diff --git a/projects/eudsl-nbgen/cmake/eudsl_nbgen-config.cmake b/projects/eudsl-nbgen/cmake/eudsl_nbgen-config.cmake index 235e8706..2548f4e5 100644 --- a/projects/eudsl-nbgen/cmake/eudsl_nbgen-config.cmake +++ b/projects/eudsl-nbgen/cmake/eudsl_nbgen-config.cmake @@ -5,25 +5,25 @@ # copy-pasta from AddMLIR.cmake/AddLLVM.cmake/TableGen.cmake -function(eudslpygen target input_file) - set(EUDSLPYGEN_TARGET_DEFINITIONS ${input_file}) - cmake_parse_arguments(ARG "" "" "DEPENDS;EXTRA_INCLUDES;NAMESPACES" ${ARGN}) - if (IS_ABSOLUTE ${EUDSLPYGEN_TARGET_DEFINITIONS}) - set(EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE ${input_file}) +function(eudsl_nbgen target input_file) + set(EUDSL_NBGEN_TARGET_DEFINITIONS ${input_file}) + cmake_parse_arguments(ARG "" "" "LINK_LIBS;EXTRA_INCLUDES;NAMESPACES" ${ARGN}) + if (IS_ABSOLUTE ${EUDSL_NBGEN_TARGET_DEFINITIONS}) + set(EUDSL_NBGEN_TARGET_DEFINITIONS_ABSOLUTE ${input_file}) else() - set(EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE ${CMAKE_CURRENT_SOURCE_DIR}/${input_file}) + set(EUDSL_NBGEN_TARGET_DEFINITIONS_ABSOLUTE ${CMAKE_CURRENT_SOURCE_DIR}/${input_file}) endif() - if(NOT EXISTS "${EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE}") + if (NOT EXISTS "${EUDSL_NBGEN_TARGET_DEFINITIONS_ABSOLUTE}") message(FATAL_ERROR "${input_file} does not exist") endif() - get_directory_property(eudslpygen_includes INCLUDE_DIRECTORIES) + get_directory_property(eudsl_nbgen_includes INCLUDE_DIRECTORIES) list(TRANSFORM ARG_EXTRA_INCLUDES PREPEND -I) - list(APPEND eudslpygen_includes ${CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES}) - list(REMOVE_ITEM eudslpygen_includes "") - list(TRANSFORM eudslpygen_includes PREPEND -I) - list(APPEND eudslpygen_includes ${ARG_EXTRA_INCLUDES}) + list(APPEND eudsl_nbgen_includes ${CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES}) + list(REMOVE_ITEM eudsl_nbgen_includes "") + list(TRANSFORM eudsl_nbgen_includes PREPEND -I) + list(APPEND eudsl_nbgen_includes ${ARG_EXTRA_INCLUDES}) set(_gen_target_dir "${CMAKE_CURRENT_BINARY_DIR}/generated/${target}") file(MAKE_DIRECTORY ${_gen_target_dir}) @@ -33,61 +33,107 @@ function(eudslpygen target input_file) # hack but most of the time we're loading headers that are downstream of tds anyway # this could be smarter by asking people to list td targets or something but that's too onerous file(GLOB_RECURSE global_tds "${MLIR_INCLUDE_DIR}/mlir/*.td") - # use cc -MM to collect all transitive headers - set(clang_command ${CMAKE_CXX_COMPILER} - # -v - -xc++ "-std=c++${CMAKE_CXX_STANDARD}" - -MM ${EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE} - -MT ${_full_gen_file} - ${eudslpygen_includes} - -o ${_depfile} - ) - execute_process(COMMAND ${clang_command} RESULT_VARIABLE _had_error_depfile - # COMMAND_ECHO STDOUT - ) + if (NOT EXISTS ${_depfile}) + # use cc -MM to collect all transitive headers + set(clang_command ${CMAKE_CXX_COMPILER} + # -v + -xc++ "-std=c++${CMAKE_CXX_STANDARD}" + -MM ${EUDSL_NBGEN_TARGET_DEFINITIONS_ABSOLUTE} + -MT ${_full_gen_file} + ${eudsl_nbgen_includes} + -o ${_depfile} + ) + execute_process(COMMAND ${clang_command} RESULT_VARIABLE _had_error_depfile + # COMMAND_ECHO STDOUT + ) + endif() - if (IS_ABSOLUTE ${EUDSLPYGEN_TARGET_DEFINITIONS}) - set(EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE ${EUDSLPYGEN_TARGET_DEFINITIONS}) + if (IS_ABSOLUTE ${EUDSL_NBGEN_TARGET_DEFINITIONS}) + set(EUDSL_NBGEN_TARGET_DEFINITIONS_ABSOLUTE ${EUDSL_NBGEN_TARGET_DEFINITIONS}) else() - set(EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE - ${CMAKE_CURRENT_SOURCE_DIR}/${EUDSLPYGEN_TARGET_DEFINITIONS}) + set(EUDSL_NBGEN_TARGET_DEFINITIONS_ABSOLUTE + ${CMAKE_CURRENT_SOURCE_DIR}/${EUDSL_NBGEN_TARGET_DEFINITIONS}) endif() - string(REPLACE " " ";" eudslpygen_defines "${LLVM_DEFINITIONS}") + string(REPLACE " " ";" eudsl_nbgen_defines "${LLVM_DEFINITIONS}") list(JOIN ARG_NAMESPACES "," namespaces) - find_program(EUDSLPY_EUDSLPYGEN_EXE "eudsl-nbgen" REQUIRED) - execute_process( - COMMAND ${EUDSLPY_EUDSLPYGEN_EXE} ${EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE} - -I${CMAKE_CURRENT_SOURCE_DIR} - -namespaces=${namespaces} - ${eudslpygen_includes} - ${eudslpygen_defines} - -o "${_full_gen_file}" - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} - RESULT_VARIABLE _had_error_gen_cpp - # COMMAND_ECHO STDOUT + set(eudsl_nbgen_generate_cmd + ${EUDSL_NBGEN_TARGET_DEFINITIONS_ABSOLUTE} + -I${CMAKE_CURRENT_LIST_DIR} -namespaces=${namespaces} + ${eudsl_nbgen_includes} ${eudsl_nbgen_defines} + -o "${_full_gen_file}" ) - if((_had_error_gen_cpp AND NOT _had_error_gen_cpp EQUAL 0) OR NOT EXISTS "${_full_gen_file}") - message(FATAL_ERROR "failed to create ${_full_gen_file}: ${_had_error_gen_cpp}") - endif() - # this is the specific thing connected the dependencies... - set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${_full_gen_file}) - execute_process( - COMMAND ${EUDSLPY_EUDSLPYGEN_EXE} -shardify ${_full_gen_file} - # ARG_EXTRA_INCLUDES has already had -I prepended - -shard-target ${target} ${ARG_EXTRA_INCLUDES} -I ${EUDSLPYGEN_TARGET_DEFINITIONS_ABSOLUTE} - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} - RESULT_VARIABLE _had_error_gen_sharded - # COMMAND_ECHO STDOUT + set(eudsl_nbgen_shardify_cmd + -shardify ${_full_gen_file} + # ARG_EXTRA_INCLUDES has already had -I prepended + -shard-target ${target} ${ARG_EXTRA_INCLUDES} -I ${EUDSL_NBGEN_TARGET_DEFINITIONS_ABSOLUTE} ) - if((_had_error_gen_sharded AND NOT _had_error_gen_sharded EQUAL 0) OR NOT EXISTS "${_full_gen_file}.sharded.cpp") - message(FATAL_ERROR "failed to create ${_full_gen_file}.sharded.cpp: ${_had_error_gen_sharded}") - endif() - set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${_full_gen_file}.sharded.cpp") - file(GLOB _shards CONFIGURE_DEPENDS "${_gen_target_dir}/*shard*cpp") - if(NOT _shards) - message(FATAL_ERROR "no shards created") + + find_program(EUDSL_NBGEN_EXE "eudsl-nbgen") + if (EUDSL_NBGEN_EXE STREQUAL "EUDSL_NBGEN_EXE-NOTFOUND") + ################################## + # not standalone build + ################################## + if (WIN32) + set(EUDSL_NBGEN_EXE "eudsl-nbgen.exe") + else() + set(EUDSL_NBGEN_EXE "eudsl-nbgen") + endif() + + string(REPLACE " " ";" eudsl_nbgen_defines "${LLVM_DEFINITIONS}") + list(JOIN ARG_NAMESPACES "," namespaces) + + add_custom_command(OUTPUT ${_full_gen_file} + COMMAND ${EUDSL_NBGEN_EXE} ${eudsl_nbgen_generate_cmd} + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + DEPENDS ${EUDSL_NBGEN_EXE} ${global_tds} + DEPFILE ${_depfile} + COMMENT "eudsl-nbgen: Generating ${_full_gen_file}..." + ) + # epic hack to specify all shards that will be generated even though we don't know them before hand + set(_shards) + # lol spirv has 260 ops + set(_max_num_shards 30) + # note this count [0, 30] <- inclusive + foreach(i RANGE ${_max_num_shards}) + list(APPEND _shards "${_full_gen_file}.shard.${i}.cpp") + endforeach() + + add_custom_command(OUTPUT "${_full_gen_file}.sharded.cpp" + COMMAND ${EUDSL_NBGEN_EXE} ${eudsl_nbgen_shardify_cmd} + -max-number-shards ${_max_num_shards} + BYPRODUCTS ${_shards} + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + DEPENDS ${_full_gen_file} ${EUDSL_NBGEN_EXE} + COMMENT "eudsl-nbgen: Generating ${_full_gen_file}.sharded.cpp..." + ) + else() + ################################## + # standalone build + ################################## + execute_process( + COMMAND ${EUDSL_NBGEN_EXE} ${eudsl_nbgen_generate_cmd} + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + RESULT_VARIABLE _had_error_gen_cpp + # COMMAND_ECHO STDOUT + ) + if ((_had_error_gen_cpp AND NOT _had_error_gen_cpp EQUAL 0) OR NOT EXISTS "${_full_gen_file}") + message(FATAL_ERROR "failed to create ${_full_gen_file}: ${_had_error_gen_cpp}") + endif() + execute_process( + COMMAND ${EUDSL_NBGEN_EXE} ${eudsl_nbgen_shardify_cmd} + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + RESULT_VARIABLE _had_error_gen_sharded + # COMMAND_ECHO STDOUT + ) + if ((_had_error_gen_sharded AND NOT _had_error_gen_sharded EQUAL 0) OR NOT EXISTS "${_full_gen_file}.sharded.cpp") + message(FATAL_ERROR "failed to create ${_full_gen_file}.sharded.cpp: ${_had_error_gen_sharded}") + endif() + file(GLOB _shards CONFIGURE_DEPENDS "${_gen_target_dir}/*shard*cpp") + if (NOT _shards) + message(FATAL_ERROR "no shards created") + endif() endif() add_library(${target} STATIC "${_full_gen_file}.sharded.cpp" ${_shards}) @@ -96,32 +142,42 @@ function(eudslpygen target input_file) OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE nanobind_include_dir RESULT_VARIABLE _has_err_find_nanobind ) - if((_has_err_find_nanobind AND NOT _has_err_find_nanobind EQUAL 0) OR NOT EXISTS "${nanobind_include_dir}") + if ((_has_err_find_nanobind AND NOT _has_err_find_nanobind EQUAL 0) OR NOT EXISTS "${nanobind_include_dir}") message(FATAL_ERROR "couldn't find nanobind include dir: ${_has_err_find_nanobind}") endif() target_include_directories(${target} PRIVATE - ${eudslpygen_includes} + ${eudsl_nbgen_includes} ${Python_INCLUDE_DIRS} ${nanobind_include_dir} ) - target_compile_options(${target} PRIVATE -Wno-cast-qual) + # not sure why unix needs this buy not apple (and really only in root-cmake build...) + if(UNIX AND NOT APPLE) + set_property(TARGET ${target} PROPERTY POSITION_INDEPENDENT_CODE ON) + endif() + set(_link_libs ${ARG_LINK_LIBS}) + if (NOT ARG_LINK_LIBS) + set(_link_libs MLIR LLVM) + endif() + target_link_libraries(${target} PUBLIC ${_link_libs}) + target_compile_options(${target} PUBLIC -Wno-cast-qual) # `make clean' must remove all those generated files: # TODO(max): clean up dep files set_property(DIRECTORY ${CMAKE_CURRENT_LIST_DIR} - APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${_shards}) + APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${_shards} ${_depfile}) set_source_files_properties(${_shards} PROPERTIES GENERATED 1) endfunction() function(patch_mlir_llvm_rpath target) + cmake_parse_arguments(ARG "STANDALONE" "" "" ${ARGN}) # hack so we can move libMLIR and libLLVM into the wheel # see AddLLVM.cmake#llvm_setup_rpath - if(APPLE OR UNIX) + if (APPLE OR UNIX) set(_origin_prefix "\$ORIGIN") - if(APPLE) + if (APPLE) set(_origin_prefix "@loader_path") endif() - if (EUDSLPY_STANDALONE_BUILD) + if (STANDALONE) get_target_property(_mlir_loc MLIR LOCATION) get_target_property(_llvm_loc LLVM LOCATION) else() @@ -129,7 +185,7 @@ function(patch_mlir_llvm_rpath target) set(_llvm_loc "$") endif() set(_old_rpath "${_origin_prefix}/../lib${LLVM_LIBDIR_SUFFIX}") - if(APPLE) + if (APPLE) if (EXISTS ${_mlir_loc}) execute_process(COMMAND install_name_tool -rpath "${_old_rpath}" ${_origin_prefix} "${_mlir_loc}" ERROR_VARIABLE rpath_err) endif() diff --git a/projects/eudsl-nbgen/src/eudsl-nbgen.cpp b/projects/eudsl-nbgen/src/eudsl-nbgen.cpp index 6db162f1..de053aac 100644 --- a/projects/eudsl-nbgen/src/eudsl-nbgen.cpp +++ b/projects/eudsl-nbgen/src/eudsl-nbgen.cpp @@ -16,8 +16,10 @@ #include "llvm/Support/ToolOutputFile.h" #include +#include #include #include +#include static llvm::cl::OptionCategory EUDSLPYGenCat("Options for eudslpy-gen"); static llvm::cl::opt InputFilename(llvm::cl::Positional, @@ -737,6 +739,14 @@ static int generate() { return 0; } +// cannot fathom why but std::to_string is return ascii characters +template +static std::string numberToString(T number) { + std::ostringstream ss; + ss << number; + return ss.str(); +} + static int makeSourceShards() { // Ensure the filename ends with ".cpp.gen" if (InputFilename.substr(InputFilename.size() - 8) != ".cpp.gen") { @@ -769,10 +779,9 @@ static int makeSourceShards() { : ShardTarget; // Generate shard files - for (size_t i = 0; i < shards.size(); ++i) { std::ofstream shardFile(InputFilename.getValue() + ".shard." + - std::to_string(i) + ".cpp"); + numberToString(i) + ".cpp"); if (!shardFile.is_open()) { llvm::errs() << "Failed to create shard file"; return -1; @@ -803,16 +812,19 @@ void populate)" << finalTarget << i << R"(Module(nb::module_ &m) { // Handle max number of shards if (MaxNumShards != -1 && shards.size() > static_cast(MaxNumShards)) { - llvm::errs() << "expected less than " + std::to_string(MaxNumShards) + - " shards"; + llvm::errs() << "expected less than " + + numberToString(MaxNumShards.getValue()) + " shards"; return -1; } if (MaxNumShards == -1) MaxNumShards = shards.size(); + else + // _max_num_shards in cmake counts +1 + MaxNumShards.getValue() += 1; for (size_t i = shards.size(); i < static_cast(MaxNumShards); ++i) { std::ofstream dummyShardFile(InputFilename.getValue() + ".shard." + - std::to_string(i) + ".cpp"); + numberToString(i) + ".cpp"); if (!dummyShardFile.is_open()) { llvm::errs() << "Failed to create dummy shard file"; return -1; diff --git a/projects/eudsl-py/CMakeLists.txt b/projects/eudsl-py/CMakeLists.txt index 7cabdb46..bad1d4cf 100644 --- a/projects/eudsl-py/CMakeLists.txt +++ b/projects/eudsl-py/CMakeLists.txt @@ -53,24 +53,6 @@ if(EUDSLPY_STANDALONE_BUILD) # for out-of-tree MLIR_INCLUDE_DIR points to the build dir by default # and MLIR_INCLUDE_DIRS points to the correct place set(MLIR_INCLUDE_DIR ${MLIR_INCLUDE_DIRS}) -else() - # turning LLVM -DLLVM_OPTIMIZED_TABLEGEN=ON builds some stuff in the NATIVE dir - # but not everything so LLVM_BINARY_DIR isn't correct - string(REPLACE "NATIVE" "" LLVM_BINARY_DIR "${LLVM_BINARY_DIR}") - # Build via external projects mechanism - set(LLVM_INCLUDE_DIR ${LLVM_MAIN_SRC_DIR}/include) - set(LLVM_GENERATED_INCLUDE_DIR ${LLVM_BINARY_DIR}/include) - set(LLVM_INCLUDE_DIRS "${LLVM_INCLUDE_DIR};${LLVM_GENERATED_INCLUDE_DIR}") - - set(MLIR_MAIN_SRC_DIR ${LLVM_MAIN_SRC_DIR}/../mlir) - set(MLIR_INCLUDE_DIR ${MLIR_MAIN_SRC_DIR}/include) - set(MLIR_GENERATED_INCLUDE_DIR ${LLVM_BINARY_DIR}/tools/mlir/include) - set(MLIR_INCLUDE_DIRS "${MLIR_INCLUDE_DIR};${MLIR_GENERATED_INCLUDE_DIR}") - - set(CLANG_MAIN_SRC_DIR ${LLVM_MAIN_SRC_DIR}/../clang) - set(CLANG_INCLUDE_DIR ${CLANG_MAIN_SRC_DIR}/include) - set(CLANG_GENERATED_INCLUDE_DIR ${LLVM_BINARY_DIR}/tools/clang/include) - set(CLANG_INCLUDE_DIRS "${CLANG_INCLUDE_DIR};${CLANG_GENERATED_INCLUDE_DIR}") endif() include_directories(${LLVM_INCLUDE_DIRS}) @@ -107,15 +89,18 @@ include_directories(${EUDSLPY_SRC_DIR}) execute_process( COMMAND "${Python_EXECUTABLE}" -m eudsl_nbgen --cmake_dir OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE eudsl_nbgen_DIR) -find_package(eudsl_nbgen CONFIG REQUIRED) +find_package(eudsl_nbgen CONFIG) +if(NOT eudsl_nbgen__FOUND) + include("${CMAKE_CURRENT_LIST_DIR}/../eudsl-nbgen/cmake/eudsl_nbgen-config.cmake") +endif() # too big -# eudslpygen(EUDSLGen_acc +# eudsl_nbgen(EUDSLGen_acc # ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h # NAMESPACES mlir::acc mlir::acc::detail # ) -eudslpygen(EUDSLGen_affine +eudsl_nbgen(EUDSLGen_affine ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h NAMESPACES mlir::affine mlir::affine::detail EXTRA_INCLUDES @@ -123,50 +108,50 @@ eudslpygen(EUDSLGen_affine mlir/Dialect/Affine/IR/AffineValueMap.h ) -eudslpygen(EUDSLGen_amdgpu +eudsl_nbgen(EUDSLGen_amdgpu ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h NAMESPACES mlir::amdgpu mlir::amdgpu::detail ) -#eudslpygen(EUDSLGen_amx +#eudsl_nbgen(EUDSLGen_amx # ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h # NAMESPACES mlir::amx mlir::amx::detail #) -eudslpygen(EUDSLGen_arith +eudsl_nbgen(EUDSLGen_arith ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h NAMESPACES mlir::arith mlir::arith::detail ) -#eudslpygen(EUDSLGen_arm_neon +#eudsl_nbgen(EUDSLGen_arm_neon # ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h # NAMESPACES mlir::arm_neon mlir::arm_neon::detail #) # too big -# eudslpygen(EUDSLGen_arm_sme +# eudsl_nbgen(EUDSLGen_arm_sme # ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h # NAMESPACES mlir::arm_sme mlir::arm_sme::detail # ) -#eudslpygen(EUDSLGen_arm_sve +#eudsl_nbgen(EUDSLGen_arm_sve # ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h # NAMESPACES mlir::arm_sve mlir::arm_sve::detail #) -eudslpygen(EUDSLGen_async +eudsl_nbgen(EUDSLGen_async ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h NAMESPACES mlir::async mlir::async::detail ) -eudslpygen(EUDSLGen_bufferization +eudsl_nbgen(EUDSLGen_bufferization ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h NAMESPACES mlir::bufferization mlir::bufferization::detail EXTRA_INCLUDES mlir/Dialect/Bufferization/Transforms/Bufferize.h ) -eudslpygen(EUDSLGen_cf +eudsl_nbgen(EUDSLGen_cf ${MLIR_INCLUDE_DIR}/mlir/Dialect/ControlFlow/IR/ControlFlowOps.h NAMESPACES mlir::cf mlir::cf::detail EXTRA_INCLUDES @@ -174,27 +159,27 @@ eudslpygen(EUDSLGen_cf mlir/IR/PatternMatch.h ) -eudslpygen(EUDSLGen_complex +eudsl_nbgen(EUDSLGen_complex ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h NAMESPACES mlir::complex mlir::complex::detail ) -#eudslpygen(EUDSLGen_DLTIDialect +#eudsl_nbgen(EUDSLGen_DLTIDialect # ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h # NAMESPACES mlir::DLTIDialect mlir::DLTIDialect::detail #) -eudslpygen(EUDSLGen_emitc +eudsl_nbgen(EUDSLGen_emitc ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h NAMESPACES mlir::emitc mlir::emitc::detail ) -eudslpygen(EUDSLGen_func +eudsl_nbgen(EUDSLGen_func ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h NAMESPACES mlir::func mlir::func::detail ) -eudslpygen(EUDSLGen_gpu +eudsl_nbgen(EUDSLGen_gpu ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h NAMESPACES mlir::gpu mlir::gpu::detail EXTRA_INCLUDES @@ -202,7 +187,7 @@ eudslpygen(EUDSLGen_gpu llvm/IR/IRBuilder.h ) -eudslpygen(EUDSLGen_index +eudsl_nbgen(EUDSLGen_index ${MLIR_INCLUDE_DIR}/mlir/Dialect/Index/IR/IndexOps.h NAMESPACES mlir::index mlir::index::detail EXTRA_INCLUDES @@ -210,76 +195,76 @@ eudslpygen(EUDSLGen_index mlir/IR/PatternMatch.h ) -#eudslpygen(EUDSLGen_irdl +#eudsl_nbgen(EUDSLGen_irdl # ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h # NAMESPACES mlir::irdl mlir::irdl::detail #) -eudslpygen(EUDSLGen_linalg +eudsl_nbgen(EUDSLGen_linalg ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h NAMESPACES mlir::linalg mlir::linalg::detail ) -eudslpygen(EUDSLGen_LLVM +eudsl_nbgen(EUDSLGen_LLVM ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h NAMESPACES mlir::LLVM mlir::LLVM::detail ) -eudslpygen(EUDSLGen_math +eudsl_nbgen(EUDSLGen_math ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h NAMESPACES mlir::math mlir::math::detail ) -eudslpygen(EUDSLGen_memref +eudsl_nbgen(EUDSLGen_memref ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h NAMESPACES mlir::memref mlir::memref::detail ) -#eudslpygen(EUDSLGen_mesh +#eudsl_nbgen(EUDSLGen_mesh # ${MLIR_INCLUDE_DIR}/mlir/Dialect/Mesh/IR/MeshOps.h # NAMESPACES mlir::mesh mlir::mesh::detail # EXTRA_INCLUDES # mlir/Dialect/Mesh/IR/MeshOps.h #) -#eudslpygen(EUDSLGen_ml_program +#eudsl_nbgen(EUDSLGen_ml_program # ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h # NAMESPACES mlir::ml_program mlir::ml_program::detail #) -#eudslpygen(EUDSLGen_mpi +#eudsl_nbgen(EUDSLGen_mpi # ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h # NAMESPACES mlir::mpi mlir::mpi::detail #) -eudslpygen(EUDSLGen_nvgpu +eudsl_nbgen(EUDSLGen_nvgpu ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h NAMESPACES mlir::nvgpu mlir::nvgpu::detail ) -eudslpygen(EUDSLGen_NVVM +eudsl_nbgen(EUDSLGen_NVVM ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h NAMESPACES mlir::NVVM mlir::NVVM::detail ) -#eudslpygen(EUDSLGen_omp +#eudsl_nbgen(EUDSLGen_omp # ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h # NAMESPACES mlir::omp mlir::omp::detail #) -eudslpygen(EUDSLGen_pdl +eudsl_nbgen(EUDSLGen_pdl ${MLIR_INCLUDE_DIR}/mlir/Dialect/PDL/IR/PDLOps.h NAMESPACES mlir::pdl mlir::pdl::detail EXTRA_INCLUDES mlir/Dialect/PDL/IR/PDLOps.h ) -eudslpygen(EUDSLGen_pdl_interp +eudsl_nbgen(EUDSLGen_pdl_interp ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h NAMESPACES mlir::pdl_interp mlir::pdl_interp::detail ) -eudslpygen(EUDSLGen_polynomial +eudsl_nbgen(EUDSLGen_polynomial ${MLIR_INCLUDE_DIR}/mlir/Dialect/Polynomial/IR/PolynomialOps.h NAMESPACES mlir::polynomial mlir::polynomial::detail EXTRA_INCLUDES @@ -287,7 +272,7 @@ eudslpygen(EUDSLGen_polynomial mlir/IR/PatternMatch.h ) -#eudslpygen(EUDSLGen_ptr +#eudsl_nbgen(EUDSLGen_ptr # ${MLIR_INCLUDE_DIR}/mlir/Dialect/Ptr/IR/PtrOps.h # NAMESPACES mlir::ptr mlir::ptr::detail # EXTRA_INCLUDES @@ -295,35 +280,35 @@ eudslpygen(EUDSLGen_polynomial # mlir/IR/DialectImplementation.h #) -#eudslpygen(EUDSLGen_quant +#eudsl_nbgen(EUDSLGen_quant # ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h # NAMESPACES mlir::quant mlir::quant::detail # EXTRA_INCLUDES # mlir/Dialect/Quant/IR/QuantTypes.h #) -eudslpygen(EUDSLGen_ROCDL +eudsl_nbgen(EUDSLGen_ROCDL ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h NAMESPACES mlir::ROCDL mlir::ROCDL::detail ) -eudslpygen(EUDSLGen_scf +eudsl_nbgen(EUDSLGen_scf ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h NAMESPACES mlir::scf mlir::scf::detail ) -eudslpygen(EUDSLGen_shape +eudsl_nbgen(EUDSLGen_shape ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h NAMESPACES mlir::shape mlir::shape::detail ) -#eudslpygen(EUDSLGen_sparse_tensor +#eudsl_nbgen(EUDSLGen_sparse_tensor # ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h # NAMESPACES mlir::sparse_tensor mlir::sparse_tensor::detail #) # nb::detail::nb_func_new("get_vce_triple_attr_name"): mismatched static/instance method flags in function overloads! -# eudslpygen(EUDSLGen_spirv +# eudsl_nbgen(EUDSLGen_spirv # ${MLIR_INCLUDE_DIR}/mlir/Dialect/SPIRV/IR/SPIRVOps.h # NAMESPACES mlir::spirv mlir::spirv::detail # EXTRA_INCLUDES @@ -331,37 +316,37 @@ eudslpygen(EUDSLGen_shape # mlir/IR/PatternMatch.h # ) -eudslpygen(EUDSLGen_tensor +eudsl_nbgen(EUDSLGen_tensor ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h NAMESPACES mlir::tensor mlir::tensor::detail ) -eudslpygen(EUDSLGen_tosa +eudsl_nbgen(EUDSLGen_tosa ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h NAMESPACES mlir::tosa mlir::tosa::detail ) -#eudslpygen(EUDSLGen_transform +#eudsl_nbgen(EUDSLGen_transform # ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h # NAMESPACES mlir::transform mlir::transform::detail #) -#eudslpygen(EUDSLGen_ub +#eudsl_nbgen(EUDSLGen_ub # ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h # NAMESPACES mlir::ub mlir::ub::detail #) -#eudslpygen(EUDSLGen_vector +#eudsl_nbgen(EUDSLGen_vector # ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h # NAMESPACES mlir::vector mlir::vector::detail #) -#eudslpygen(EUDSLGen_x86vector +#eudsl_nbgen(EUDSLGen_x86vector # ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h # NAMESPACES mlir::x86vector mlir::x86vector::detail #) -#eudslpygen(EUDSLGen_xegpu +#eudsl_nbgen(EUDSLGen_xegpu # ${MLIR_INCLUDE_DIR}/mlir/InitAllDialects.h # NAMESPACES mlir::xegpu mlir::xegpu::detail #) @@ -425,14 +410,17 @@ set_target_properties(eudslpy_ext PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${EUDSLPY_SRC_DIR}/eudsl" ) -target_link_libraries(eudslpy_ext PRIVATE MLIR) # hack - on GHA, linux, the build OOMs set(EUDSLPY_DISABLE_COMPILE_OPT ON CACHE BOOL "") if (EUDSLPY_DISABLE_COMPILE_OPT) target_compile_options(eudslpy_ext PRIVATE -O0) endif() -patch_mlir_llvm_rpath(eudslpy_ext) +set(STANDALONE) +if(EUDSLPY_STANDALONE_BUILD OR EUDSL_STANDALONE_BUILD) + set(STANDALONE STANDALONE) +endif() +patch_mlir_llvm_rpath(eudslpy_ext ${STANDALONE}) # copy libMLIR into the ext dir for wheels install(IMPORTED_RUNTIME_ARTIFACTS MLIR LLVM LIBRARY DESTINATION eudsl) diff --git a/projects/eudsl-tblgen/CMakeLists.txt b/projects/eudsl-tblgen/CMakeLists.txt index d8ff2530..0396b6ea 100644 --- a/projects/eudsl-tblgen/CMakeLists.txt +++ b/projects/eudsl-tblgen/CMakeLists.txt @@ -4,17 +4,40 @@ # Copyright (c) 2024. cmake_minimum_required(VERSION 3.29) -project(eudsl_tblgen CXX C) set(CMAKE_CXX_STANDARD 17) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +set(LLVM_SUBPROJECT_TITLE "EUDSL_TBLGEN") +set(EUDSL_TBLGEN_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}) + +if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_LIST_DIR) + message("Building ${LLVM_SUBPROJECT_TITLE} as a standalone project.") + project(${LLVM_SUBPROJECT_TITLE} CXX C) + find_package(LLVM REQUIRED CONFIG) + + message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") + + set(LLVM_RUNTIME_OUTPUT_INTDIR ${CMAKE_BINARY_DIR}/bin) + set(LLVM_LIBRARY_OUTPUT_INTDIR ${CMAKE_BINARY_DIR}/lib) + + list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_DIR}") + + include(TableGen) + include(AddLLVM) + # TODO(max): probably don't need this anymore after landing the nanobind fix? + # technically we need this on windows too but our LLVM is compiled without exception handling + # and that breaks windows + if(NOT WIN32) + include(HandleLLVMOptions) + endif() +endif() -find_package(LLVM REQUIRED CONFIG) -message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") include_directories(${LLVM_INCLUDE_DIRS}) +link_directories(${LLVM_BUILD_LIBRARY_DIR}) +add_definitions(${LLVM_DEFINITIONS}) -# technically we need this but our LLVM is compiled without exception handling -# and that breaks windows -if(NOT WIN32) - include(HandleLLVMOptions) +if(NOT TARGET LLVMSupport) + message(FATAL_ERROR "LLVMSupport not found") endif() find_package(Python 3.8 @@ -26,7 +49,7 @@ execute_process( OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE nanobind_DIR) find_package(nanobind CONFIG REQUIRED) -set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/eudsl_tblgen) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/eudsl_tblgen) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) nanobind_add_module(eudsl_tblgen_ext NB_STATIC STABLE_ABI @@ -37,6 +60,7 @@ nanobind_add_module(eudsl_tblgen_ext NB_STATIC STABLE_ABI target_link_libraries(eudsl_tblgen_ext PRIVATE LLVMTableGenCommon LLVMTableGen) target_compile_options(eudsl_tblgen_ext PUBLIC + -Wno-cast-qual $<$:-fexceptions -frtti> $<$:-fexceptions -frtti> $<$:/EHsc /GR>) @@ -44,21 +68,21 @@ target_compile_options(eudsl_tblgen_ext nanobind_add_stub( eudsl_tblgen_ext_stub MODULE eudsl_tblgen_ext - OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/eudsl_tblgen/eudsl_tblgen_ext.pyi + OUTPUT ${CMAKE_CURRENT_LIST_DIR}/eudsl_tblgen/eudsl_tblgen_ext.pyi PYTHON_PATH $ DEPENDS eudsl_tblgen_ext ) nanobind_add_stub( eudsl_tblgen_stub MODULE eudsl_tblgen - OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/eudsl_tblgen/__init__.pyi - PYTHON_PATH ${CMAKE_CURRENT_SOURCE_DIR} + OUTPUT ${CMAKE_CURRENT_LIST_DIR}/eudsl_tblgen/__init__.pyi + PYTHON_PATH ${CMAKE_CURRENT_LIST_DIR} DEPENDS eudsl_tblgen_ext ) install(TARGETS eudsl_tblgen_ext LIBRARY DESTINATION eudsl_tblgen) install( - DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/eudsl_tblgen + DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/eudsl_tblgen DESTINATION ${CMAKE_INSTALL_PREFIX} PATTERN "*.so" EXCLUDE PATTERN "*.a" EXCLUDE diff --git a/projects/eudsl-tblgen/requirements.txt b/projects/eudsl-tblgen/requirements.txt deleted file mode 100644 index 1f942f7e..00000000 --- a/projects/eudsl-tblgen/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -click==8.1.7 -pytest==8.3.3 -nanobind==2.2.0 diff --git a/projects/eudsl-py/requirements.txt b/requirements.txt similarity index 67% rename from projects/eudsl-py/requirements.txt rename to requirements.txt index 003b5eee..53635869 100644 --- a/projects/eudsl-py/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ -pytest==8.3.3 nanobind==2.4.0 numpy==2.0.2