CMake Superbuild: Tarball (#1668)

Cloning whole repos can be quite slow and CMake does not support shallow clones yet. We can use release tarballs instead.
openPMD · Sep 18, 2024 · 88089bc · 88089bc
1 parent ff0c669
commit 88089bc
Show file tree

Hide file tree

Showing 6 changed files with 124 additions and 35 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -12,7 +12,11 @@ include(${openPMD_SOURCE_DIR}/cmake/openPMDFunctions.cmake)
 
 # CMake policies ##############################################################
 #
-# not needed right now
+# CMake 3.24+ tarball download robustness
+#   https://cmake.org/cmake/help/latest/module/ExternalProject.html#url
+if(POLICY CMP0135)
+    cmake_policy(SET CMP0135 NEW)
+endif()
 
 
 # No in-Source builds #########################################################
@@ -566,14 +570,8 @@ if(openPMD_HAVE_PYTHON)
         endif()
     endif()
     message(STATUS "Python LTO/IPO: ${_USE_PY_LTO}")
-    if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.18)
-        set_target_properties(openPMD.py PROPERTIES
-            INTERPROCEDURAL_OPTIMIZATION ${_USE_PY_LTO})
-    else()
-        if(_USE_PY_LTO)
-            target_link_libraries(openPMD.py PRIVATE pybind11::lto)
-        endif()
-    endif()
+    set_target_properties(openPMD.py PROPERTIES
+        INTERPROCEDURAL_OPTIMIZATION ${_USE_PY_LTO})
     unset(_USE_PY_LTO)
 
     if(EMSCRIPTEN)

diff --git a/cmake/dependencies/catch.cmake b/cmake/dependencies/catch.cmake
@@ -7,22 +7,33 @@ function(find_catch2)
         if(NOT IS_DIRECTORY ${openPMD_catch_src})
             message(FATAL_ERROR "Specified directory openPMD_catch_src='${openPMD_catch_src}' does not exist!")
         endif()
+    elseif(openPMD_catch_tar)
+        message(STATUS "Downloading Catch2 ...")
+        message(STATUS "Catch2 source: ${openPMD_catch_tar}")
     elseif(openPMD_USE_INTERNAL_CATCH)
         message(STATUS "Downloading Catch2 ...")
         message(STATUS "Catch2 repository: ${openPMD_catch_repo} (${openPMD_catch_branch})")
-        include(FetchContent)
     endif()
     if(TARGET Catch2::Catch2)
         # nothing to do, target already exists in the superbuild
-    elseif(openPMD_USE_INTERNAL_CATCH OR openPMD_catch_src)
+    elseif(openPMD_USE_INTERNAL_CATCH OR openPMD_catch_src OR openPMD_catch_tar)
         if(openPMD_catch_src)
             add_subdirectory(${openPMD_catch_src} _deps/localCatch2-build/)
         else()
-            FetchContent_Declare(fetchedCatch2
-                GIT_REPOSITORY ${openPMD_catch_repo}
-                GIT_TAG        ${openPMD_catch_branch}
-                BUILD_IN_SOURCE 0
-            )
+            include(FetchContent)
+            if(openPMD_catch_tar)
+                FetchContent_Declare(fetchedCatch2
+                    URL             ${openPMD_catch_tar}
+                    URL_HASH        ${openPMD_catch_tar_hash}
+                    BUILD_IN_SOURCE OFF
+                )
+            else()
+                FetchContent_Declare(fetchedCatch2
+                    GIT_REPOSITORY ${openPMD_catch_repo}
+                    GIT_TAG        ${openPMD_catch_branch}
+                    BUILD_IN_SOURCE OFF
+                )
+            endif()
             FetchContent_MakeAvailable(fetchedCatch2)
 
             # advanced fetch options
@@ -44,6 +55,14 @@ set(openPMD_catch_src ""
     CACHE PATH
     "Local path to Catch2 source directory (preferred if set)")
 
+# tarball fetcher
+set(openPMD_catch_tar "https://github.com/catchorg/Catch2/archive/refs/tags/v2.13.10.tar.gz"
+    CACHE STRING
+    "Remote tarball link to pull and build Catch2 from if(openPMD_USE_INTERNAL_CATCH)")
+set(openPMD_catch_tar_hash "SHA256=d54a712b7b1d7708bc7a819a8e6e47b2fde9536f487b89ccbca295072a7d9943"
+    CACHE STRING
+    "Hash checksum of the tarball of Catch2 if(openPMD_USE_INTERNAL_CATCH)")
+
 # Git fetcher
 set(openPMD_catch_repo "https://github.com/catchorg/Catch2.git"
     CACHE STRING

diff --git a/cmake/dependencies/json.cmake b/cmake/dependencies/json.cmake
@@ -7,22 +7,33 @@ function(find_json)
         if(NOT IS_DIRECTORY ${openPMD_json_src})
             message(FATAL_ERROR "Specified directory openPMD_json_src='${openPMD_json_src}' does not exist!")
         endif()
+    elseif(openPMD_json_tar)
+        message(STATUS "Downloading nlohmann_json ...")
+        message(STATUS "nlohmann_json source: ${openPMD_json_tar}")
     elseif(openPMD_USE_INTERNAL_JSON)
         message(STATUS "Downloading nlohmann_json ...")
         message(STATUS "nlohmann_json repository: ${openPMD_json_repo} (${openPMD_json_branch})")
-        include(FetchContent)
     endif()
     if(TARGET nlohmann_json::nlohmann_json)
         # nothing to do, target already exists in the superbuild
-    elseif(openPMD_USE_INTERNAL_JSON OR openPMD_json_src)
+    elseif(openPMD_USE_INTERNAL_JSON OR openPMD_json_src OR openPMD_json_tar)
         if(openPMD_json_src)
             add_subdirectory(${openPMD_json_src} _deps/localnlohmann_json-build/)
         else()
-            FetchContent_Declare(fetchednlohmann_json
-                GIT_REPOSITORY ${openPMD_json_repo}
-                GIT_TAG        ${openPMD_json_branch}
-                BUILD_IN_SOURCE 0
-            )
+            include(FetchContent)
+            if(openPMD_json_tar)
+                FetchContent_Declare(fetchednlohmann_json
+                    URL             ${openPMD_json_tar}
+                    URL_HASH        ${openPMD_json_tar_hash}
+                    BUILD_IN_SOURCE OFF
+                )
+            else()
+                FetchContent_Declare(fetchednlohmann_json
+                    GIT_REPOSITORY ${openPMD_json_repo}
+                    GIT_TAG        ${openPMD_json_branch}
+                    BUILD_IN_SOURCE OFF
+                )
+            endif()
             FetchContent_MakeAvailable(fetchednlohmann_json)
 
             # advanced fetch options
@@ -44,6 +55,14 @@ set(openPMD_json_src ""
     CACHE PATH
     "Local path to nlohmann_json source directory (preferred if set)")
 
+# tarball fetcher
+set(openPMD_json_tar "https://github.com/nlohmann/json/archive/refs/tags/v3.11.3.tar.gz"
+    CACHE STRING
+    "Remote tarball link to pull and build nlohmann_json from if(openPMD_USE_INTERNAL_JSON)")
+set(openPMD_json_tar_hash "SHA256=0d8ef5af7f9794e3263480193c491549b2ba6cc74bb018906202ada498a79406"
+    CACHE STRING
+    "Hash checksum of the tarball of nlohmann_json if(openPMD_USE_INTERNAL_JSON)")
+
 # Git fetcher
 set(openPMD_json_repo "https://github.com/nlohmann/json.git"
     CACHE STRING

diff --git a/cmake/dependencies/pybind11.cmake b/cmake/dependencies/pybind11.cmake
@@ -7,22 +7,33 @@ function(find_pybind11)
         if(NOT IS_DIRECTORY ${openPMD_pybind11_src})
             message(FATAL_ERROR "Specified directory openPMD_pybind11_src='${openPMD_pybind11_src}' does not exist!")
         endif()
+    elseif(openPMD_pybind11_tar)
+        message(STATUS "Downloading pybind11 ...")
+        message(STATUS "pybind11 source: ${openPMD_pybind11_tar}")
     elseif(openPMD_USE_INTERNAL_PYBIND11)
         message(STATUS "Downloading pybind11 ...")
         message(STATUS "pybind11 repository: ${openPMD_pybind11_repo} (${openPMD_pybind11_branch})")
-        include(FetchContent)
     endif()
     if(TARGET pybind11::module)
         # nothing to do, target already exists in the superbuild
     elseif(openPMD_USE_INTERNAL_PYBIND11 OR openPMD_pybind11_src)
         if(openPMD_pybind11_src)
             add_subdirectory(${openPMD_pybind11_src} _deps/localpybind11-build/)
         else()
-            FetchContent_Declare(fetchedpybind11
-                GIT_REPOSITORY ${openPMD_pybind11_repo}
-                GIT_TAG        ${openPMD_pybind11_branch}
-                BUILD_IN_SOURCE 0
-            )
+            include(FetchContent)
+            if(openPMD_pybind11_tar)
+                FetchContent_Declare(fetchedpybind11
+                        URL             ${openPMD_pybind11_tar}
+                        URL_HASH        ${openPMD_pybind11_tar_hash}
+                        BUILD_IN_SOURCE OFF
+                )
+            else()
+                FetchContent_Declare(fetchedpybind11
+                    GIT_REPOSITORY ${openPMD_pybind11_repo}
+                    GIT_TAG        ${openPMD_pybind11_branch}
+                    BUILD_IN_SOURCE OFF
+                )
+            endif()
             FetchContent_MakeAvailable(fetchedpybind11)
 
             # advanced fetch options
@@ -50,6 +61,14 @@ set(openPMD_pybind11_src ""
     CACHE PATH
     "Local path to pybind11 source directory (preferred if set)")
 
+# tarball fetcher
+set(openPMD_pybind11_tar "https://github.com/pybind/pybind11/archive/refs/tags/v2.12.0.tar.gz"
+        CACHE STRING
+        "Remote tarball link to pull and build pybind11 from if(openPMD_USE_INTERNAL_PYBIND11)")
+set(openPMD_pybind11_tar_hash "SHA256=bf8f242abd1abcd375d516a7067490fb71abd79519a282d22b6e4d19282185a7"
+        CACHE STRING
+        "Hash checksum of the tarball of pybind11 if(openPMD_USE_INTERNAL_PYBIND11)")
+
 # Git fetcher
 set(openPMD_pybind11_repo "https://github.com/pybind/pybind11.git"
     CACHE STRING

diff --git a/cmake/dependencies/toml11.cmake b/cmake/dependencies/toml11.cmake
@@ -7,22 +7,33 @@ function(find_toml11)
         if(NOT IS_DIRECTORY ${openPMD_toml11_src})
             message(FATAL_ERROR "Specified directory openPMD_toml11_src='${openPMD_toml11_src}' does not exist!")
         endif()
+    elseif(openPMD_toml11_tar)
+        message(STATUS "Downloading toml11 ...")
+        message(STATUS "toml11 source: ${openPMD_toml11_tar}")
     elseif(openPMD_USE_INTERNAL_TOML11)
         message(STATUS "Downloading toml11 ...")
         message(STATUS "toml11 repository: ${openPMD_toml11_repo} (${openPMD_toml11_branch})")
-        include(FetchContent)
     endif()
     if(TARGET toml11::toml11)
         # nothing to do, target already exists in the superbuild
     elseif(openPMD_USE_INTERNAL_TOML11 OR openPMD_toml11_src)
         if(openPMD_toml11_src)
             add_subdirectory(${openPMD_toml11_src} _deps/localtoml11-build/)
         else()
-            FetchContent_Declare(fetchedtoml11
-                GIT_REPOSITORY ${openPMD_toml11_repo}
-                GIT_TAG        ${openPMD_toml11_branch}
-                BUILD_IN_SOURCE 0
-            )
+            include(FetchContent)
+            if(openPMD_toml11_tar)
+                FetchContent_Declare(fetchedtoml11
+                        URL             ${openPMD_toml11_tar}
+                        URL_HASH        ${openPMD_toml11_tar_hash}
+                        BUILD_IN_SOURCE OFF
+                )
+            else()
+                FetchContent_Declare(fetchedtoml11
+                    GIT_REPOSITORY ${openPMD_toml11_repo}
+                    GIT_TAG        ${openPMD_toml11_branch}
+                    BUILD_IN_SOURCE OFF
+                )
+            endif()
             FetchContent_MakeAvailable(fetchedtoml11)
 
             # advanced fetch options
@@ -52,6 +63,14 @@ set(openPMD_toml11_src ""
     CACHE PATH
     "Local path to toml11 source directory (preferred if set)")
 
+# tarball fetcher
+set(openPMD_toml11_tar "https://github.com/ToruNiina/toml11/archive/refs/tags/v4.2.0.tar.gz"
+        CACHE STRING
+        "Remote tarball link to pull and build toml11 from if(openPMD_USE_INTERNAL_TOML11)")
+set(openPMD_toml11_tar_hash "SHA256=9287971cd4a1a3992ef37e7b95a3972d1ae56410e7f8e3f300727ab1d6c79c2c"
+        CACHE STRING
+        "Hash checksum of the tarball of toml11 if(openPMD_USE_INTERNAL_TOML11)")
+
 # Git fetcher
 set(openPMD_toml11_repo "https://github.com/ToruNiina/toml11.git"
     CACHE STRING

diff --git a/docs/source/dev/buildoptions.rst b/docs/source/dev/buildoptions.rst
@@ -80,19 +80,34 @@ Developers can also use a local copy of the source directories or control the ex
 CMake Option                  Default & Values                               Description
 ============================= ============================================== ===========================================================
 ``openPMD_catch_src``         *None*                                         Path to Catch2 source directory (preferred if set)
+``openPMD_catch_tar``         *we set and maintain a compatible version*     Remote tarball link to pull and build Catch2 from
+``openPMD_catch_tar_hash``    *we set and maintain a compatible version*     Hash checksum of the tarball of Catch2
 ``openPMD_catch_repo``        ``https://github.com/catchorg/Catch2.git``     Repository URI to pull and build Catch2 from
 ``openPMD_catch_branch``      *we set and maintain a compatible commit*      Repository branch for ``openPMD_catch_repo``
 ``openPMD_pybind11_src``      *None*                                         Path to pybind11 source directory (preferred if set)
+``openPMD_pybind11_tar``      *we set and maintain a compatible version*     Remote tarball link to pull and build pybind11 from
+``openPMD_pybind11_tar_hash`` *we set and maintain a compatible version*     Hash checksum of the tarball of pybind11
 ``openPMD_pybind11_repo``     ``https://github.com/pybind/pybind11.git``     Repository URI to pull and build pybind11 from
 ``openPMD_pybind11_branch``   *we set and maintain a compatible commit*      Repository branch for ``openPMD_pybind11_repo``
 ``openPMD_json_src``          *None*                                         Path to NLohmann JSON source directory (preferred if set)
+``openPMD_json_tar``          *we set and maintain a compatible version*     Remote tarball link to pull and build NLohmann JSON from
+``openPMD_json_tar_hash``     *we set and maintain a compatible version*     Hash checksum of the tarball of NLohmann JSON
 ``openPMD_json_repo``         ``https://github.com/nlohmann/json.git``       Repository URI to pull and build NLohmann JSON from
 ``openPMD_json_branch``       *we set and maintain a compatible commit*      Repository branch for ``openPMD_json_repo``
 ``openPMD_toml11_src``        *None*                                         Path to TOML11 source directory (preferred if set)
+``openPMD_toml11_tar``        *we set and maintain a compatible version*     Remote tarball link to pull and build TOML11 from
+``openPMD_toml11_tar_hash``   *we set and maintain a compatible version*     Hash checksum of the tarball of TOML11
 ``openPMD_toml11_repo``       ``https://github.com/ToruNiina/toml11.git``    Repository URI to pull and build TOML11 from
 ``openPMD_toml11_branch``     *we set and maintain a compatible commit*      Repository branch for ``openPMD_toml11_repo``
 ============================= ============================================== ===========================================================
 
+The precedence for these options is:
+
+#. existing targets found in a superbuild (from another project)
+#. ``_src`` is specified
+#. ``_tar`` is specified
+#. ``_repo`` + ``_branch`` are specified
+
 
 Tests, Examples and Command Line Tools
 --------------------------------------