Updated conan to install zstd/1.5.2 & zlib/1.2.11 to successfully bui…

…ld BundledRocksDB

 On a fresh dev Ubuntu 22.04 docker environment that mainly comes with the apt packages needed
to build MiNiFi, I realized that there were problems with compilation coming from BundledRocksDB,

so once I updated the BundledRocksDB to temporarily account for conan packages zstd & zlib,

  BundledRocksDB built rocksdb system library. When building MiNiFi with conan, I later plan
to switch from installing system library rocksdb to installing the conan package rocksdb with

 the needed patches. We were able to successfully install rocksdb sysem library when building

  MiNiFi with conan build and MiNiFi then built successfully using conan build.

    However, two ctests failed '34 - ProvenanceTests (Subprocess aborted)' and
'219 - ControllerTests (Failed)'.

  I didn't see the compilation failure for MiNiFi conan build related to

'librdkafka/kafka-external-prefix/src/kafka-external/src/rdcrc32.h:57:10' for

'fatal error: zlib.h: No such file or directory'.

cmake/BundledRocksDB.cmake

@@ -19,7 +19,8 @@ function(use_bundled_rocksdb SOURCE_DIR BINARY_DIR)
     message("Using bundled RocksDB")
 
     if (NOT WIN32)
-        include(Zstd)
+        include(GetZstd)
+        get_zstd()
         list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/zstd/dummy")
 
         include(LZ4)
@@ -63,10 +64,17 @@ function(use_bundled_rocksdb SOURCE_DIR BINARY_DIR)
                 -DROCKSDB_INSTALL_ON_WINDOWS=ON
                 -DWITH_XPRESS=ON)
     else()
+    if(MINIFI_ZSTD_SOURCE STREQUAL "CONAN" AND MINIFI_ZLIB_SOURCE STREQUAL "CONAN")
+        list(APPEND ROCKSDB_CMAKE_ARGS
+            -DWITH_ZLIB=OFF
+            -DWITH_ZSTD=OFF)
+    else()
+        list(APPEND ROCKSDB_CMAKE_ARGS
+            -DWITH_ZLIB=ON
+            -DWITH_ZSTD=ON)
+    endif()
         list(APPEND ROCKSDB_CMAKE_ARGS
-                -DWITH_ZLIB=ON
                 -DWITH_BZ2=ON
-                -DWITH_ZSTD=ON
                 -DWITH_LZ4=ON)
     endif()
 
@@ -95,7 +103,11 @@ function(use_bundled_rocksdb SOURCE_DIR BINARY_DIR)
     add_library(RocksDB::RocksDB STATIC IMPORTED)
     set_target_properties(RocksDB::RocksDB PROPERTIES IMPORTED_LOCATION "${ROCKSDB_LIBRARY}")
     if (NOT WIN32)
-        add_dependencies(rocksdb-external ZLIB::ZLIB BZip2::BZip2 zstd::zstd lz4::lz4)
+        if(MINIFI_ZSTD_SOURCE STREQUAL "CONAN" AND MINIFI_ZLIB_SOURCE STREQUAL "CONAN")
+            add_dependencies(rocksdb-external BZip2::BZip2 lz4::lz4)
+        else()
+            add_dependencies(rocksdb-external ZLIB::ZLIB BZip2::BZip2 zstd::zstd lz4::lz4)
+        endif()
     endif()
     add_dependencies(RocksDB::RocksDB rocksdb-external)
     file(MAKE_DIRECTORY ${ROCKSDB_INCLUDE_DIR})

cmake/GetZstd.cmake

@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+function(get_zstd)
+    if(MINIFI_ZSTD_SOURCE STREQUAL "CONAN")
+        message("Using Conan Packager to manage installing prebuilt zstd external lib")
+        find_package(zstd REQUIRED)
+        add_library(zstd::zstd ALIAS zstd::libzstd_static)
+    elseif(MINIFI_ZSTD_SOURCE STREQUAL "BUILD")
+        message("Using CMAKE's ExternalProject_Add to manage source building zstd external lib")
+        include(Zstd)
+    endif()
+endfunction(get_zstd)

cmake/MiNiFiOptions.cmake

@@ -139,6 +139,7 @@ set_property(CACHE STRICT_GSL_CHECKS PROPERTY STRINGS ${STRICT_GSL_CHECKS_Values
 add_minifi_multi_option(MINIFI_LIBCURL_SOURCE "Retrieves LibCURL from provided source" "BUILD;SYSTEM;CONAN" "BUILD")
 add_minifi_multi_option(MINIFI_OPENSSL_SOURCE "Retrieves OpenSSL from provided source" "BUILD;SYSTEM;CONAN" "BUILD")
 add_minifi_multi_option(MINIFI_ZLIB_SOURCE "Retrieves ZLib from provided source" "BUILD;SYSTEM;CONAN" "BUILD")
+add_minifi_multi_option(MINIFI_ZSTD_SOURCE "Retrieves Zstd from provided source" "BUILD;SYSTEM;CONAN" "BUILD")
 add_minifi_multi_option(MINIFI_CIVETWEB_SOURCE "Retrieves CivetWeb from provided source" "BUILD;SYSTEM;CONAN" "BUILD")
 add_minifi_multi_option(MINIFI_LIBXML2_SOURCE "Retrieves LibXml2 from provided source" "BUILD;SYSTEM;CONAN" "BUILD")
 add_minifi_multi_option(MINIFI_FMT_SOURCE "Retrieves Fmt from provided source" "BUILD;SYSTEM;CONAN" "BUILD")

conanfile.py

@@ -7,7 +7,7 @@
 
 required_conan_version = ">=2.0"
 
-shared_requires = ("openssl/3.2.1", "libcurl/8.6.0", "civetweb/1.16", "libxml2/2.12.6", "fmt/10.2.1", "spdlog/1.14.0", "catch2/3.5.4", "zlib/1.2.11")
+shared_requires = ("openssl/3.2.1", "libcurl/8.4.0", "civetweb/1.16", "libxml2/2.12.6", "fmt/10.2.1", "spdlog/1.14.0", "catch2/3.5.4", "zlib/1.2.11", "zstd/1.5.2")
 
 shared_sources = ("CMakeLists.txt", "libminifi/*", "extensions/*", "minifi_main/*", "nanofi/*", "bin/*", "bootstrap/*", "cmake/*", "conf/*", "controller/*", "encrypt-config/*", "etc/*", "examples/*", "msi/*", "thirdparty/*", "docker/*", "LICENSE", "NOTICE", "README.md", "C2.md", "CONFIGURE.md", "CONTRIBUTING.md", "CONTROLLERS.md", "EXPRESSIONS.md", "Extensions.md", "JNI.md", "METRICS.md", "OPS.md", "PROCESSORS.md", "ThirdParties.md", "Windows.md", "aptitude.sh", "arch.sh", "bootstrap.sh", "bstrp_functions.sh", "centos.sh", "CPPLINT.cfg", "darwin.sh", "debian.sh", "deploy.sh", "fedora.sh", "generateVersion.sh", "linux.sh", "rheldistro.sh", "run_clang_tidy.sh", "run_clang_tidy.sh", "run_flake8.sh", "run_shellcheck.sh", "suse.sh", "versioninfo.rc.in")
 
@@ -31,6 +31,7 @@ def generate(self):
         tc.variables["MINIFI_LIBCURL_SOURCE"] = "CONAN"
         tc.variables["MINIFI_OPENSSL_SOURCE"] = "CONAN"
         tc.variables["MINIFI_ZLIB_SOURCE"] = "CONAN"
+        tc.variables["MINIFI_ZSTD_SOURCE"] = "CONAN"
         tc.variables["MINIFI_CIVETWEB_SOURCE"] = "CONAN"
         tc.variables["MINIFI_LIBXML2_SOURCE"] = "CONAN"
         tc.variables["MINIFI_FMT_SOURCE"] = "CONAN"