From 88089bcc0edef65d9d5a566a13aeb3a78346de43 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 18 Sep 2024 09:57:48 -0700 Subject: [PATCH] CMake Superbuild: Tarball (#1668) Cloning whole repos can be quite slow and CMake does not support shallow clones yet. We can use release tarballs instead. --- CMakeLists.txt | 16 +++++++-------- cmake/dependencies/catch.cmake | 33 ++++++++++++++++++++++++------- cmake/dependencies/json.cmake | 33 ++++++++++++++++++++++++------- cmake/dependencies/pybind11.cmake | 31 +++++++++++++++++++++++------ cmake/dependencies/toml11.cmake | 31 +++++++++++++++++++++++------ docs/source/dev/buildoptions.rst | 15 ++++++++++++++ 6 files changed, 124 insertions(+), 35 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 80811d4cb2..a0cc161f4a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,11 @@ include(${openPMD_SOURCE_DIR}/cmake/openPMDFunctions.cmake) # CMake policies ############################################################## # -# not needed right now +# CMake 3.24+ tarball download robustness +# https://cmake.org/cmake/help/latest/module/ExternalProject.html#url +if(POLICY CMP0135) + cmake_policy(SET CMP0135 NEW) +endif() # No in-Source builds ######################################################### @@ -566,14 +570,8 @@ if(openPMD_HAVE_PYTHON) endif() endif() message(STATUS "Python LTO/IPO: ${_USE_PY_LTO}") - if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.18) - set_target_properties(openPMD.py PROPERTIES - INTERPROCEDURAL_OPTIMIZATION ${_USE_PY_LTO}) - else() - if(_USE_PY_LTO) - target_link_libraries(openPMD.py PRIVATE pybind11::lto) - endif() - endif() + set_target_properties(openPMD.py PROPERTIES + INTERPROCEDURAL_OPTIMIZATION ${_USE_PY_LTO}) unset(_USE_PY_LTO) if(EMSCRIPTEN) diff --git a/cmake/dependencies/catch.cmake b/cmake/dependencies/catch.cmake index 5c600ba5bc..956d66493e 100644 --- a/cmake/dependencies/catch.cmake +++ b/cmake/dependencies/catch.cmake @@ -7,22 +7,33 @@ function(find_catch2) if(NOT IS_DIRECTORY ${openPMD_catch_src}) message(FATAL_ERROR "Specified directory openPMD_catch_src='${openPMD_catch_src}' does not exist!") endif() + elseif(openPMD_catch_tar) + message(STATUS "Downloading Catch2 ...") + message(STATUS "Catch2 source: ${openPMD_catch_tar}") elseif(openPMD_USE_INTERNAL_CATCH) message(STATUS "Downloading Catch2 ...") message(STATUS "Catch2 repository: ${openPMD_catch_repo} (${openPMD_catch_branch})") - include(FetchContent) endif() if(TARGET Catch2::Catch2) # nothing to do, target already exists in the superbuild - elseif(openPMD_USE_INTERNAL_CATCH OR openPMD_catch_src) + elseif(openPMD_USE_INTERNAL_CATCH OR openPMD_catch_src OR openPMD_catch_tar) if(openPMD_catch_src) add_subdirectory(${openPMD_catch_src} _deps/localCatch2-build/) else() - FetchContent_Declare(fetchedCatch2 - GIT_REPOSITORY ${openPMD_catch_repo} - GIT_TAG ${openPMD_catch_branch} - BUILD_IN_SOURCE 0 - ) + include(FetchContent) + if(openPMD_catch_tar) + FetchContent_Declare(fetchedCatch2 + URL ${openPMD_catch_tar} + URL_HASH ${openPMD_catch_tar_hash} + BUILD_IN_SOURCE OFF + ) + else() + FetchContent_Declare(fetchedCatch2 + GIT_REPOSITORY ${openPMD_catch_repo} + GIT_TAG ${openPMD_catch_branch} + BUILD_IN_SOURCE OFF + ) + endif() FetchContent_MakeAvailable(fetchedCatch2) # advanced fetch options @@ -44,6 +55,14 @@ set(openPMD_catch_src "" CACHE PATH "Local path to Catch2 source directory (preferred if set)") +# tarball fetcher +set(openPMD_catch_tar "https://github.com/catchorg/Catch2/archive/refs/tags/v2.13.10.tar.gz" + CACHE STRING + "Remote tarball link to pull and build Catch2 from if(openPMD_USE_INTERNAL_CATCH)") +set(openPMD_catch_tar_hash "SHA256=d54a712b7b1d7708bc7a819a8e6e47b2fde9536f487b89ccbca295072a7d9943" + CACHE STRING + "Hash checksum of the tarball of Catch2 if(openPMD_USE_INTERNAL_CATCH)") + # Git fetcher set(openPMD_catch_repo "https://github.com/catchorg/Catch2.git" CACHE STRING diff --git a/cmake/dependencies/json.cmake b/cmake/dependencies/json.cmake index 2e144caadd..8b22eb80f8 100644 --- a/cmake/dependencies/json.cmake +++ b/cmake/dependencies/json.cmake @@ -7,22 +7,33 @@ function(find_json) if(NOT IS_DIRECTORY ${openPMD_json_src}) message(FATAL_ERROR "Specified directory openPMD_json_src='${openPMD_json_src}' does not exist!") endif() + elseif(openPMD_json_tar) + message(STATUS "Downloading nlohmann_json ...") + message(STATUS "nlohmann_json source: ${openPMD_json_tar}") elseif(openPMD_USE_INTERNAL_JSON) message(STATUS "Downloading nlohmann_json ...") message(STATUS "nlohmann_json repository: ${openPMD_json_repo} (${openPMD_json_branch})") - include(FetchContent) endif() if(TARGET nlohmann_json::nlohmann_json) # nothing to do, target already exists in the superbuild - elseif(openPMD_USE_INTERNAL_JSON OR openPMD_json_src) + elseif(openPMD_USE_INTERNAL_JSON OR openPMD_json_src OR openPMD_json_tar) if(openPMD_json_src) add_subdirectory(${openPMD_json_src} _deps/localnlohmann_json-build/) else() - FetchContent_Declare(fetchednlohmann_json - GIT_REPOSITORY ${openPMD_json_repo} - GIT_TAG ${openPMD_json_branch} - BUILD_IN_SOURCE 0 - ) + include(FetchContent) + if(openPMD_json_tar) + FetchContent_Declare(fetchednlohmann_json + URL ${openPMD_json_tar} + URL_HASH ${openPMD_json_tar_hash} + BUILD_IN_SOURCE OFF + ) + else() + FetchContent_Declare(fetchednlohmann_json + GIT_REPOSITORY ${openPMD_json_repo} + GIT_TAG ${openPMD_json_branch} + BUILD_IN_SOURCE OFF + ) + endif() FetchContent_MakeAvailable(fetchednlohmann_json) # advanced fetch options @@ -44,6 +55,14 @@ set(openPMD_json_src "" CACHE PATH "Local path to nlohmann_json source directory (preferred if set)") +# tarball fetcher +set(openPMD_json_tar "https://github.com/nlohmann/json/archive/refs/tags/v3.11.3.tar.gz" + CACHE STRING + "Remote tarball link to pull and build nlohmann_json from if(openPMD_USE_INTERNAL_JSON)") +set(openPMD_json_tar_hash "SHA256=0d8ef5af7f9794e3263480193c491549b2ba6cc74bb018906202ada498a79406" + CACHE STRING + "Hash checksum of the tarball of nlohmann_json if(openPMD_USE_INTERNAL_JSON)") + # Git fetcher set(openPMD_json_repo "https://github.com/nlohmann/json.git" CACHE STRING diff --git a/cmake/dependencies/pybind11.cmake b/cmake/dependencies/pybind11.cmake index 82b97f8c97..0fb6209a53 100644 --- a/cmake/dependencies/pybind11.cmake +++ b/cmake/dependencies/pybind11.cmake @@ -7,10 +7,12 @@ function(find_pybind11) if(NOT IS_DIRECTORY ${openPMD_pybind11_src}) message(FATAL_ERROR "Specified directory openPMD_pybind11_src='${openPMD_pybind11_src}' does not exist!") endif() + elseif(openPMD_pybind11_tar) + message(STATUS "Downloading pybind11 ...") + message(STATUS "pybind11 source: ${openPMD_pybind11_tar}") elseif(openPMD_USE_INTERNAL_PYBIND11) message(STATUS "Downloading pybind11 ...") message(STATUS "pybind11 repository: ${openPMD_pybind11_repo} (${openPMD_pybind11_branch})") - include(FetchContent) endif() if(TARGET pybind11::module) # nothing to do, target already exists in the superbuild @@ -18,11 +20,20 @@ function(find_pybind11) if(openPMD_pybind11_src) add_subdirectory(${openPMD_pybind11_src} _deps/localpybind11-build/) else() - FetchContent_Declare(fetchedpybind11 - GIT_REPOSITORY ${openPMD_pybind11_repo} - GIT_TAG ${openPMD_pybind11_branch} - BUILD_IN_SOURCE 0 - ) + include(FetchContent) + if(openPMD_pybind11_tar) + FetchContent_Declare(fetchedpybind11 + URL ${openPMD_pybind11_tar} + URL_HASH ${openPMD_pybind11_tar_hash} + BUILD_IN_SOURCE OFF + ) + else() + FetchContent_Declare(fetchedpybind11 + GIT_REPOSITORY ${openPMD_pybind11_repo} + GIT_TAG ${openPMD_pybind11_branch} + BUILD_IN_SOURCE OFF + ) + endif() FetchContent_MakeAvailable(fetchedpybind11) # advanced fetch options @@ -50,6 +61,14 @@ set(openPMD_pybind11_src "" CACHE PATH "Local path to pybind11 source directory (preferred if set)") +# tarball fetcher +set(openPMD_pybind11_tar "https://github.com/pybind/pybind11/archive/refs/tags/v2.12.0.tar.gz" + CACHE STRING + "Remote tarball link to pull and build pybind11 from if(openPMD_USE_INTERNAL_PYBIND11)") +set(openPMD_pybind11_tar_hash "SHA256=bf8f242abd1abcd375d516a7067490fb71abd79519a282d22b6e4d19282185a7" + CACHE STRING + "Hash checksum of the tarball of pybind11 if(openPMD_USE_INTERNAL_PYBIND11)") + # Git fetcher set(openPMD_pybind11_repo "https://github.com/pybind/pybind11.git" CACHE STRING diff --git a/cmake/dependencies/toml11.cmake b/cmake/dependencies/toml11.cmake index 5f5bc7a684..49cc3e97cb 100644 --- a/cmake/dependencies/toml11.cmake +++ b/cmake/dependencies/toml11.cmake @@ -7,10 +7,12 @@ function(find_toml11) if(NOT IS_DIRECTORY ${openPMD_toml11_src}) message(FATAL_ERROR "Specified directory openPMD_toml11_src='${openPMD_toml11_src}' does not exist!") endif() + elseif(openPMD_toml11_tar) + message(STATUS "Downloading toml11 ...") + message(STATUS "toml11 source: ${openPMD_toml11_tar}") elseif(openPMD_USE_INTERNAL_TOML11) message(STATUS "Downloading toml11 ...") message(STATUS "toml11 repository: ${openPMD_toml11_repo} (${openPMD_toml11_branch})") - include(FetchContent) endif() if(TARGET toml11::toml11) # nothing to do, target already exists in the superbuild @@ -18,11 +20,20 @@ function(find_toml11) if(openPMD_toml11_src) add_subdirectory(${openPMD_toml11_src} _deps/localtoml11-build/) else() - FetchContent_Declare(fetchedtoml11 - GIT_REPOSITORY ${openPMD_toml11_repo} - GIT_TAG ${openPMD_toml11_branch} - BUILD_IN_SOURCE 0 - ) + include(FetchContent) + if(openPMD_toml11_tar) + FetchContent_Declare(fetchedtoml11 + URL ${openPMD_toml11_tar} + URL_HASH ${openPMD_toml11_tar_hash} + BUILD_IN_SOURCE OFF + ) + else() + FetchContent_Declare(fetchedtoml11 + GIT_REPOSITORY ${openPMD_toml11_repo} + GIT_TAG ${openPMD_toml11_branch} + BUILD_IN_SOURCE OFF + ) + endif() FetchContent_MakeAvailable(fetchedtoml11) # advanced fetch options @@ -52,6 +63,14 @@ set(openPMD_toml11_src "" CACHE PATH "Local path to toml11 source directory (preferred if set)") +# tarball fetcher +set(openPMD_toml11_tar "https://github.com/ToruNiina/toml11/archive/refs/tags/v4.2.0.tar.gz" + CACHE STRING + "Remote tarball link to pull and build toml11 from if(openPMD_USE_INTERNAL_TOML11)") +set(openPMD_toml11_tar_hash "SHA256=9287971cd4a1a3992ef37e7b95a3972d1ae56410e7f8e3f300727ab1d6c79c2c" + CACHE STRING + "Hash checksum of the tarball of toml11 if(openPMD_USE_INTERNAL_TOML11)") + # Git fetcher set(openPMD_toml11_repo "https://github.com/ToruNiina/toml11.git" CACHE STRING diff --git a/docs/source/dev/buildoptions.rst b/docs/source/dev/buildoptions.rst index 21a8ed0ec6..8473d4cc7d 100644 --- a/docs/source/dev/buildoptions.rst +++ b/docs/source/dev/buildoptions.rst @@ -80,19 +80,34 @@ Developers can also use a local copy of the source directories or control the ex CMake Option Default & Values Description ============================= ============================================== =========================================================== ``openPMD_catch_src`` *None* Path to Catch2 source directory (preferred if set) +``openPMD_catch_tar`` *we set and maintain a compatible version* Remote tarball link to pull and build Catch2 from +``openPMD_catch_tar_hash`` *we set and maintain a compatible version* Hash checksum of the tarball of Catch2 ``openPMD_catch_repo`` ``https://github.com/catchorg/Catch2.git`` Repository URI to pull and build Catch2 from ``openPMD_catch_branch`` *we set and maintain a compatible commit* Repository branch for ``openPMD_catch_repo`` ``openPMD_pybind11_src`` *None* Path to pybind11 source directory (preferred if set) +``openPMD_pybind11_tar`` *we set and maintain a compatible version* Remote tarball link to pull and build pybind11 from +``openPMD_pybind11_tar_hash`` *we set and maintain a compatible version* Hash checksum of the tarball of pybind11 ``openPMD_pybind11_repo`` ``https://github.com/pybind/pybind11.git`` Repository URI to pull and build pybind11 from ``openPMD_pybind11_branch`` *we set and maintain a compatible commit* Repository branch for ``openPMD_pybind11_repo`` ``openPMD_json_src`` *None* Path to NLohmann JSON source directory (preferred if set) +``openPMD_json_tar`` *we set and maintain a compatible version* Remote tarball link to pull and build NLohmann JSON from +``openPMD_json_tar_hash`` *we set and maintain a compatible version* Hash checksum of the tarball of NLohmann JSON ``openPMD_json_repo`` ``https://github.com/nlohmann/json.git`` Repository URI to pull and build NLohmann JSON from ``openPMD_json_branch`` *we set and maintain a compatible commit* Repository branch for ``openPMD_json_repo`` ``openPMD_toml11_src`` *None* Path to TOML11 source directory (preferred if set) +``openPMD_toml11_tar`` *we set and maintain a compatible version* Remote tarball link to pull and build TOML11 from +``openPMD_toml11_tar_hash`` *we set and maintain a compatible version* Hash checksum of the tarball of TOML11 ``openPMD_toml11_repo`` ``https://github.com/ToruNiina/toml11.git`` Repository URI to pull and build TOML11 from ``openPMD_toml11_branch`` *we set and maintain a compatible commit* Repository branch for ``openPMD_toml11_repo`` ============================= ============================================== =========================================================== +The precedence for these options is: + +#. existing targets found in a superbuild (from another project) +#. ``_src`` is specified +#. ``_tar`` is specified +#. ``_repo`` + ``_branch`` are specified + Tests, Examples and Command Line Tools --------------------------------------