From a34b2e01c6777106fffd81f9bc7436f452a1d4c7 Mon Sep 17 00:00:00 2001 From: james94 Date: Mon, 2 Sep 2024 10:55:42 -0700 Subject: [PATCH] Updated conan to install zstd/1.5.2 & zlib/1.2.11 to successfully build BundledRocksDB On a fresh dev Ubuntu 22.04 docker environment that mainly comes with the apt packages needed to build MiNiFi, I realized that there were problems with compilation coming from BundledRocksDB, so once I updated the BundledRocksDB to temporarily account for conan packages zstd & zlib, BundledRocksDB built rocksdb system library. When building MiNiFi with conan, I later plan to switch from installing system library rocksdb to installing the conan package rocksdb with the needed patches. We were able to successfully install rocksdb sysem library when building MiNiFi with conan build and MiNiFi then built successfully using conan build. However, two ctests failed '34 - ProvenanceTests (Subprocess aborted)' and '219 - ControllerTests (Failed)'. I didn't see the compilation failure for MiNiFi conan build related to 'librdkafka/kafka-external-prefix/src/kafka-external/src/rdcrc32.h:57:10' for 'fatal error: zlib.h: No such file or directory'. --- cmake/BundledRocksDB.cmake | 20 ++++++++++++++++---- cmake/GetZstd.cmake | 27 +++++++++++++++++++++++++++ cmake/MiNiFiOptions.cmake | 1 + conanfile.py | 3 ++- 4 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 cmake/GetZstd.cmake diff --git a/cmake/BundledRocksDB.cmake b/cmake/BundledRocksDB.cmake index 2d4211e944..34882f26de 100644 --- a/cmake/BundledRocksDB.cmake +++ b/cmake/BundledRocksDB.cmake @@ -19,7 +19,8 @@ function(use_bundled_rocksdb SOURCE_DIR BINARY_DIR) message("Using bundled RocksDB") if (NOT WIN32) - include(Zstd) + include(GetZstd) + get_zstd() list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/zstd/dummy") include(LZ4) @@ -63,10 +64,17 @@ function(use_bundled_rocksdb SOURCE_DIR BINARY_DIR) -DROCKSDB_INSTALL_ON_WINDOWS=ON -DWITH_XPRESS=ON) else() + if(MINIFI_ZSTD_SOURCE STREQUAL "CONAN" AND MINIFI_ZLIB_SOURCE STREQUAL "CONAN") + list(APPEND ROCKSDB_CMAKE_ARGS + -DWITH_ZLIB=OFF + -DWITH_ZSTD=OFF) + else() + list(APPEND ROCKSDB_CMAKE_ARGS + -DWITH_ZLIB=ON + -DWITH_ZSTD=ON) + endif() list(APPEND ROCKSDB_CMAKE_ARGS - -DWITH_ZLIB=ON -DWITH_BZ2=ON - -DWITH_ZSTD=ON -DWITH_LZ4=ON) endif() @@ -95,7 +103,11 @@ function(use_bundled_rocksdb SOURCE_DIR BINARY_DIR) add_library(RocksDB::RocksDB STATIC IMPORTED) set_target_properties(RocksDB::RocksDB PROPERTIES IMPORTED_LOCATION "${ROCKSDB_LIBRARY}") if (NOT WIN32) - add_dependencies(rocksdb-external ZLIB::ZLIB BZip2::BZip2 zstd::zstd lz4::lz4) + if(MINIFI_ZSTD_SOURCE STREQUAL "CONAN" AND MINIFI_ZLIB_SOURCE STREQUAL "CONAN") + add_dependencies(rocksdb-external BZip2::BZip2 lz4::lz4) + else() + add_dependencies(rocksdb-external ZLIB::ZLIB BZip2::BZip2 zstd::zstd lz4::lz4) + endif() endif() add_dependencies(RocksDB::RocksDB rocksdb-external) file(MAKE_DIRECTORY ${ROCKSDB_INCLUDE_DIR}) diff --git a/cmake/GetZstd.cmake b/cmake/GetZstd.cmake new file mode 100644 index 0000000000..2b23929531 --- /dev/null +++ b/cmake/GetZstd.cmake @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function(get_zstd) + if(MINIFI_ZSTD_SOURCE STREQUAL "CONAN") + message("Using Conan Packager to manage installing prebuilt zstd external lib") + find_package(zstd REQUIRED) + add_library(zstd::zstd ALIAS zstd::libzstd_static) + elseif(MINIFI_ZSTD_SOURCE STREQUAL "BUILD") + message("Using CMAKE's ExternalProject_Add to manage source building zstd external lib") + include(Zstd) + endif() +endfunction(get_zstd) diff --git a/cmake/MiNiFiOptions.cmake b/cmake/MiNiFiOptions.cmake index 545829d86b..dac88ce852 100644 --- a/cmake/MiNiFiOptions.cmake +++ b/cmake/MiNiFiOptions.cmake @@ -139,6 +139,7 @@ set_property(CACHE STRICT_GSL_CHECKS PROPERTY STRINGS ${STRICT_GSL_CHECKS_Values add_minifi_multi_option(MINIFI_LIBCURL_SOURCE "Retrieves LibCURL from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_OPENSSL_SOURCE "Retrieves OpenSSL from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_ZLIB_SOURCE "Retrieves ZLib from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_ZSTD_SOURCE "Retrieves Zstd from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_CIVETWEB_SOURCE "Retrieves CivetWeb from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_LIBXML2_SOURCE "Retrieves LibXml2 from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_FMT_SOURCE "Retrieves Fmt from provided source" "BUILD;SYSTEM;CONAN" "BUILD") diff --git a/conanfile.py b/conanfile.py index 2d12c8e568..0f4f82b27b 100644 --- a/conanfile.py +++ b/conanfile.py @@ -7,7 +7,7 @@ required_conan_version = ">=2.0" -shared_requires = ("openssl/3.2.1", "libcurl/8.6.0", "civetweb/1.16", "libxml2/2.12.6", "fmt/10.2.1", "spdlog/1.14.0", "catch2/3.5.4", "zlib/1.2.11") +shared_requires = ("openssl/3.2.1", "libcurl/8.4.0", "civetweb/1.16", "libxml2/2.12.6", "fmt/10.2.1", "spdlog/1.14.0", "catch2/3.5.4", "zlib/1.2.11", "zstd/1.5.2") shared_sources = ("CMakeLists.txt", "libminifi/*", "extensions/*", "minifi_main/*", "nanofi/*", "bin/*", "bootstrap/*", "cmake/*", "conf/*", "controller/*", "encrypt-config/*", "etc/*", "examples/*", "msi/*", "thirdparty/*", "docker/*", "LICENSE", "NOTICE", "README.md", "C2.md", "CONFIGURE.md", "CONTRIBUTING.md", "CONTROLLERS.md", "EXPRESSIONS.md", "Extensions.md", "JNI.md", "METRICS.md", "OPS.md", "PROCESSORS.md", "ThirdParties.md", "Windows.md", "aptitude.sh", "arch.sh", "bootstrap.sh", "bstrp_functions.sh", "centos.sh", "CPPLINT.cfg", "darwin.sh", "debian.sh", "deploy.sh", "fedora.sh", "generateVersion.sh", "linux.sh", "rheldistro.sh", "run_clang_tidy.sh", "run_clang_tidy.sh", "run_flake8.sh", "run_shellcheck.sh", "suse.sh", "versioninfo.rc.in") @@ -31,6 +31,7 @@ def generate(self): tc.variables["MINIFI_LIBCURL_SOURCE"] = "CONAN" tc.variables["MINIFI_OPENSSL_SOURCE"] = "CONAN" tc.variables["MINIFI_ZLIB_SOURCE"] = "CONAN" + tc.variables["MINIFI_ZSTD_SOURCE"] = "CONAN" tc.variables["MINIFI_CIVETWEB_SOURCE"] = "CONAN" tc.variables["MINIFI_LIBXML2_SOURCE"] = "CONAN" tc.variables["MINIFI_FMT_SOURCE"] = "CONAN"