Created RocksDB Conan Recipe to take patches like BundledRocksDB

Also when we create the RocksDB conan package, it not only builds with the patches like BundledRocksDB, it also incorporates ZLib and ZStd conan packages too. Therefore, now that we use a conan RocksDB for building MiNiFi with conan, when we build MiNiFi with standalone CMake, we use the original version of BundledRocksDB that was in 'main' branch before I modified it. I verified that when I build MiNiFi with conan, 218 CTESTs pass out of 219. Similarly, when I build MiNiFi with standalone, then I run CTESTs, 218 out of 219 pass too.
apache · Oct 2, 2024 · 71686e3 · 71686e3
1 parent a34b2e0
commit 71686e3
Show file tree

Hide file tree

Showing 23 changed files with 369 additions and 46 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -228,8 +228,10 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/ssl")
 
 
 if (ENABLE_BZIP2 AND (ENABLE_LIBARCHIVE OR (ENABLE_ROCKSDB AND NOT WIN32)))
-    include(BundledBZip2)
-    use_bundled_bzip2(${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR})
+    include(GetBZip2)
+    get_bzip2(${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR})
+    # include(BundledBZip2)
+    # use_bundled_bzip2(${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR})
     list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/bzip2/dummy")
 endif()
 

diff --git a/CONAN.md b/CONAN.md
@@ -25,6 +25,7 @@ We will walk through the steps to build MiNiFi using conan version 2 that comes
   - [Build MiNiFi C++ with Conan](#build-minifi-c---with-conan)
   - [Create MiNiFi C++ Conan Package](#create-minifi-c---conan-package)
   - [Conclusion](#conclusion)
+  - [Appendix](#appendix)
 
 ## Description
 
@@ -120,3 +121,15 @@ To have a more consistent quick build process for MiNiFi, we can use conan versi
 There are multiple benefits of having MiNiFi prebuilt conan packages. We can upload these MiNiFi conan packages to a conan repository like jfrog for version management. We can easily integrate MiNiFi's edge data pipeline features into other C++ software infrastructure using conan's CMake support. We can still use MiNiFi for edge data collection from the IoT devices embedded on robotic systems. We can integrate MiNiFi into self-driving cars (sensor examples: cameras, lidar, radar, inertial measurement unit (IMU), electronic speed controller (ESC), steering servo, etc), into medical imaging robots (sensor examples: depth cameras, ultrasound, gamma detector, force/torque sensor, joint position sensor, etc) or some other real-time robotic system.
 
 By leveraging MiNiFi as a conan package, we can leverage MiNiFi that comes with the best practices of building data pipelines from NiFi and bring them into existing C++ real-time robotics infrastructure. Some teams across companies typically have their own custom edge data pipelines that process data for the different events to eventually perform actions on that data. As an alternative to all these companies and their teams having their own custom edge data pipeline libraries, MiNiFi C++, which is like a headless NiFi, can provide a more consistent standard approach for team's to build edge pipelines. Through all stages of the edge data pipelines, MiNiFi can still provide telemetry to NiFi instances running in the cloud.
+
+## Appendix
+
+### Create Custom RocksDB Conan Package
+
+~~~bash
+pushd nifi-minifi-cpp/thirdparty/rocksdb/all
+
+conan create . --user=minifi --channel=develop --version=8.10.2 -pr=../../../etc/conan/profiles/release-linux
+
+popd
+~~~
diff --git a/cmake/BundledRocksDB.cmake b/cmake/BundledRocksDB.cmake
@@ -19,18 +19,17 @@ function(use_bundled_rocksdb SOURCE_DIR BINARY_DIR)
     message("Using bundled RocksDB")
 
     if (NOT WIN32)
-        include(GetZstd)
-        get_zstd()
+        include(Zstd)
         list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/zstd/dummy")
 
         include(LZ4)
         list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/lz4/dummy")
     endif()
 
     # Patch to fix build issue on ARM7 architecture: https://github.com/facebook/rocksdb/issues/8609#issuecomment-1009572506
-    set(PATCH_FILE_1 "${SOURCE_DIR}/thirdparty/rocksdb/arm7.patch")
-    set(PATCH_FILE_2 "${SOURCE_DIR}/thirdparty/rocksdb/dboptions_equality_operator.patch")
-    set(PATCH_FILE_3 "${SOURCE_DIR}/thirdparty/rocksdb/cstdint.patch")
+    set(PATCH_FILE_1 "${SOURCE_DIR}/thirdparty/rocksdb/all/patches/arm7.patch")
+    set(PATCH_FILE_2 "${SOURCE_DIR}/thirdparty/rocksdb/all/patches/dboptions_equality_operator.patch")
+    set(PATCH_FILE_3 "${SOURCE_DIR}/thirdparty/rocksdb/all/patches/cstdint.patch")
     set(PC ${Bash_EXECUTABLE} -c "set -x &&\
             (\"${Patch_EXECUTABLE}\" -p1 -R -s -f --dry-run -i \"${PATCH_FILE_1}\" || \"${Patch_EXECUTABLE}\" -p1 -N -i \"${PATCH_FILE_1}\") &&\
             (\"${Patch_EXECUTABLE}\" -p1 -R -s -f --dry-run -i \"${PATCH_FILE_2}\" || \"${Patch_EXECUTABLE}\" -p1 -N -i \"${PATCH_FILE_2}\") &&\
@@ -64,17 +63,10 @@ function(use_bundled_rocksdb SOURCE_DIR BINARY_DIR)
                 -DROCKSDB_INSTALL_ON_WINDOWS=ON
                 -DWITH_XPRESS=ON)
     else()
-    if(MINIFI_ZSTD_SOURCE STREQUAL "CONAN" AND MINIFI_ZLIB_SOURCE STREQUAL "CONAN")
-        list(APPEND ROCKSDB_CMAKE_ARGS
-            -DWITH_ZLIB=OFF
-            -DWITH_ZSTD=OFF)
-    else()
-        list(APPEND ROCKSDB_CMAKE_ARGS
-            -DWITH_ZLIB=ON
-            -DWITH_ZSTD=ON)
-    endif()
         list(APPEND ROCKSDB_CMAKE_ARGS
+                -DWITH_ZLIB=ON
                 -DWITH_BZ2=ON
+                -DWITH_ZSTD=ON
                 -DWITH_LZ4=ON)
     endif()
 
@@ -103,11 +95,7 @@ function(use_bundled_rocksdb SOURCE_DIR BINARY_DIR)
     add_library(RocksDB::RocksDB STATIC IMPORTED)
     set_target_properties(RocksDB::RocksDB PROPERTIES IMPORTED_LOCATION "${ROCKSDB_LIBRARY}")
     if (NOT WIN32)
-        if(MINIFI_ZSTD_SOURCE STREQUAL "CONAN" AND MINIFI_ZLIB_SOURCE STREQUAL "CONAN")
-            add_dependencies(rocksdb-external BZip2::BZip2 lz4::lz4)
-        else()
-            add_dependencies(rocksdb-external ZLIB::ZLIB BZip2::BZip2 zstd::zstd lz4::lz4)
-        endif()
+        add_dependencies(rocksdb-external ZLIB::ZLIB BZip2::BZip2 zstd::zstd lz4::lz4)
     endif()
     add_dependencies(RocksDB::RocksDB rocksdb-external)
     file(MAKE_DIRECTORY ${ROCKSDB_INCLUDE_DIR})

diff --git a/cmake/GetBZip2.cmake b/cmake/GetBZip2.cmake
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+function(get_bzip2 SOURCE_DIR BINARY_DIR)
+    if(MINIFI_BZIP2_SOURCE STREQUAL "CONAN")
+        message("Using Conan Packager to manage installing prebuilt bzip2 external lib")
+        find_package(BZip2 REQUIRED)
+    elseif(MINIFI_BZIP2_SOURCE STREQUAL "BUILD")
+        message("Using CMake to build bzip2 from source")
+        include(BundledBZip2)
+        use_bundled_bzip2(${SOURCE_DIR} ${BINARY_DIR})
+    endif()
+endfunction(get_bzip2)
diff --git a/cmake/GetCatch2.cmake b/cmake/GetCatch2.cmake
@@ -21,7 +21,7 @@ function(get_catch2)
         find_package(Catch2 REQUIRED)
         add_library(Catch2WithMain ALIAS Catch2::Catch2WithMain)
     elseif(MINIFI_CATCH2_SOURCE STREQUAL "BUILD")
-        message("Using CMAKE's ExternalProject_Add to manage source building Catch2 external lib")
+        message("Using CMake to build Catch2 from source")
         include(Catch2)
     endif()
 endfunction(get_catch2)
diff --git a/cmake/GetCivetWeb.cmake b/cmake/GetCivetWeb.cmake
@@ -20,7 +20,7 @@ function(get_civetweb)
         message("Using Conan Packager to manage installing prebuilt CivetWeb external lib")
         find_package(civetweb REQUIRED)
     elseif(MINIFI_CIVETWEB_SOURCE STREQUAL "BUILD")
-        message("Using CMAKE's ExternalProject_Add to manage source building CivetWeb external lib")
+        message("Using CMake to build CivetWeb from source")
         include(CivetWeb)
     endif()
 endfunction(get_civetweb)
diff --git a/cmake/GetFmt.cmake b/cmake/GetFmt.cmake
@@ -20,7 +20,7 @@ function(get_fmt)
         message("Using Conan Packager to manage installing prebuilt Fmt external lib")
         find_package(fmt REQUIRED)
     elseif(MINIFI_FMT_SOURCE STREQUAL "BUILD")
-        message("Using CMAKE's ExternalProject_Add to manage source building Fmt external lib")
+        message("Using CMake to build Fmt from source")
         include(fmt)
     endif()
 endfunction(get_fmt)
diff --git a/cmake/GetLibCURL.cmake b/cmake/GetLibCURL.cmake
@@ -20,7 +20,7 @@ function(get_curl SOURCE_DIR BINARY_DIR)
         message("Using Conan Packager to manage installing prebuilt libcurl external lib")
         find_package(CURL REQUIRED)
     elseif(MINIFI_LIBCURL_SOURCE STREQUAL "BUILD")
-        message("Using CMAKE's ExternalProject_Add to manage source building libcurl external lib")
+        message("Using CMake to build libcurl from source")
         include(BundledLibcURL)
         use_bundled_curl(${SOURCE_DIR} ${BINARY_DIR})
     endif()

diff --git a/cmake/GetLibXml2.cmake b/cmake/GetLibXml2.cmake
@@ -20,7 +20,7 @@ function(get_libxml2 SOURCE_DIR BINARY_DIR)
         message("Using Conan Packager to manage installing prebuilt LibXml2 external lib")
         find_package(libxml2 REQUIRED)
     elseif(MINIFI_LIBXML2_SOURCE STREQUAL "BUILD")
-        message("Using CMAKE's ExternalProject_Add to manage source building LibXml2 external lib")
+        message("Using CMake to build LibXml2 from source")
         include(BundledLibXml2)
         use_bundled_libxml2(${SOURCE_DIR} ${BINARY_DIR})
     endif()

diff --git a/cmake/GetOpenSSL.cmake b/cmake/GetOpenSSL.cmake
@@ -20,7 +20,7 @@ function(get_openssl SOURCE_DIR BINARY_DIR)
         message("Using Conan Packager to manage installing prebuilt OpenSSL external lib")
         find_package(OpenSSL REQUIRED)
     elseif(MINIFI_OPENSSL_SOURCE STREQUAL "BUILD")
-        message("Using CMAKE's ExternalProject_Add to manage source building OpenSSL external lib")
+        message("Using CMake to build OpenSSL from source")
         include(BundledOpenSSL)
         use_openssl(${SOURCE_DIR} ${BINARY_DIR})
     endif()

diff --git a/cmake/GetRocksDB.cmake b/cmake/GetRocksDB.cmake
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+function(get_rocksdb SOURCE_DIR BINARY_DIR)
+    if(MINIFI_ROCKSDB_SOURCE STREQUAL "CONAN")
+        message("Using Conan Packager to manage installing prebuilt RocksDB external lib")
+        find_package(RocksDB REQUIRED)
+        add_library(RocksDB::RocksDB ALIAS RocksDB::rocksdb)
+    elseif(MINIFI_ROCKSDB_SOURCE STREQUAL "BUILD")
+        message("Using CMake to build RocksDB from source")
+
+        if (BUILD_ROCKSDB)
+            include(BundledRocksDB)
+            use_bundled_rocksdb(${SOURCE_DIR} ${BINARY_DIR})
+        else()
+            list(APPEND CMAKE_MODULE_PATH "${SOURCE_DIR}/cmake/rocksdb/sys")
+            find_package(RocksDB REQUIRED)
+        endif()
+    endif()
+endfunction(get_rocksdb SOURCE_DIR BINARY_DIR)
diff --git a/cmake/GetSpdlog.cmake b/cmake/GetSpdlog.cmake
@@ -25,7 +25,7 @@ function(get_spdlog)
 
         add_library(spdlog ALIAS spdlog::spdlog)
     elseif(MINIFI_SPDLOG_SOURCE STREQUAL "BUILD")
-        message("Using CMAKE's ExternalProject_Add to manage source building Spdlog external lib")
+        message("Using CMake to build Spdlog from source")
         include(Spdlog)
     endif()
 endfunction(get_spdlog)
diff --git a/cmake/GetZLIB.cmake b/cmake/GetZLIB.cmake
@@ -20,7 +20,7 @@ function(get_zlib SOURCE_DIR BINARY_DIR)
         message("Using Conan Packager to manage installing prebuilt zlib external lib")
         find_package(ZLIB REQUIRED)
     elseif(MINIFI_ZLIB_SOURCE STREQUAL "BUILD")
-        message("Using CMAKE's ExternalProject_Add to manage source building zlib external lib")
+        message("Using CMake to build zlib from source")
         include(BundledZLIB)
         use_bundled_zlib(${SOURCE_DIR} ${BINARY_DIR})
     endif()

diff --git a/cmake/GetZstd.cmake b/cmake/GetZstd.cmake
@@ -21,7 +21,7 @@ function(get_zstd)
         find_package(zstd REQUIRED)
         add_library(zstd::zstd ALIAS zstd::libzstd_static)
     elseif(MINIFI_ZSTD_SOURCE STREQUAL "BUILD")
-        message("Using CMAKE's ExternalProject_Add to manage source building zstd external lib")
+        message("Using CMake to build zstd from source")
         include(Zstd)
     endif()
 endfunction(get_zstd)
diff --git a/cmake/MiNiFiOptions.cmake b/cmake/MiNiFiOptions.cmake
@@ -139,7 +139,9 @@ set_property(CACHE STRICT_GSL_CHECKS PROPERTY STRINGS ${STRICT_GSL_CHECKS_Values
 add_minifi_multi_option(MINIFI_LIBCURL_SOURCE "Retrieves LibCURL from provided source" "BUILD;SYSTEM;CONAN" "BUILD")
 add_minifi_multi_option(MINIFI_OPENSSL_SOURCE "Retrieves OpenSSL from provided source" "BUILD;SYSTEM;CONAN" "BUILD")
 add_minifi_multi_option(MINIFI_ZLIB_SOURCE "Retrieves ZLib from provided source" "BUILD;SYSTEM;CONAN" "BUILD")
+add_minifi_multi_option(MINIFI_ROCKSDB_SOURCE "Retrieves RocksDB from provided source" "BUILD;SYSTEM;CONAN" "BUILD")
 add_minifi_multi_option(MINIFI_ZSTD_SOURCE "Retrieves Zstd from provided source" "BUILD;SYSTEM;CONAN" "BUILD")
+add_minifi_multi_option(MINIFI_BZIP2_SOURCE "Retrieves BZip2 from provided source" "BUILD;SYSTEM;CONAN" "BUILD")
 add_minifi_multi_option(MINIFI_CIVETWEB_SOURCE "Retrieves CivetWeb from provided source" "BUILD;SYSTEM;CONAN" "BUILD")
 add_minifi_multi_option(MINIFI_LIBXML2_SOURCE "Retrieves LibXml2 from provided source" "BUILD;SYSTEM;CONAN" "BUILD")
 add_minifi_multi_option(MINIFI_FMT_SOURCE "Retrieves Fmt from provided source" "BUILD;SYSTEM;CONAN" "BUILD")

diff --git a/conanfile.py b/conanfile.py
@@ -7,7 +7,7 @@
 
 required_conan_version = ">=2.0"
 
-shared_requires = ("openssl/3.2.1", "libcurl/8.4.0", "civetweb/1.16", "libxml2/2.12.6", "fmt/10.2.1", "spdlog/1.14.0", "catch2/3.5.4", "zlib/1.2.11", "zstd/1.5.2")
+shared_requires = ("openssl/3.2.1", "libcurl/8.4.0", "civetweb/1.16", "libxml2/2.12.6", "fmt/10.2.1", "spdlog/1.14.0", "catch2/3.5.4", "zlib/1.2.11", "zstd/1.5.2", "bzip2/1.0.8", "rocksdb/8.10.2@minifi/develop")
 
 shared_sources = ("CMakeLists.txt", "libminifi/*", "extensions/*", "minifi_main/*", "nanofi/*", "bin/*", "bootstrap/*", "cmake/*", "conf/*", "controller/*", "encrypt-config/*", "etc/*", "examples/*", "msi/*", "thirdparty/*", "docker/*", "LICENSE", "NOTICE", "README.md", "C2.md", "CONFIGURE.md", "CONTRIBUTING.md", "CONTROLLERS.md", "EXPRESSIONS.md", "Extensions.md", "JNI.md", "METRICS.md", "OPS.md", "PROCESSORS.md", "ThirdParties.md", "Windows.md", "aptitude.sh", "arch.sh", "bootstrap.sh", "bstrp_functions.sh", "centos.sh", "CPPLINT.cfg", "darwin.sh", "debian.sh", "deploy.sh", "fedora.sh", "generateVersion.sh", "linux.sh", "rheldistro.sh", "run_clang_tidy.sh", "run_clang_tidy.sh", "run_flake8.sh", "run_shellcheck.sh", "suse.sh", "versioninfo.rc.in")
 
@@ -31,7 +31,9 @@ def generate(self):
         tc.variables["MINIFI_LIBCURL_SOURCE"] = "CONAN"
         tc.variables["MINIFI_OPENSSL_SOURCE"] = "CONAN"
         tc.variables["MINIFI_ZLIB_SOURCE"] = "CONAN"
+        tc.variables["MINIFI_ROCKSDB_SOURCE"] = "CONAN"
         tc.variables["MINIFI_ZSTD_SOURCE"] = "CONAN"
+        tc.variables["MINIFI_BZIP2_SOURCE"] = "CONAN"
         tc.variables["MINIFI_CIVETWEB_SOURCE"] = "CONAN"
         tc.variables["MINIFI_LIBXML2_SOURCE"] = "CONAN"
         tc.variables["MINIFI_FMT_SOURCE"] = "CONAN"

diff --git a/extensions/rocksdb-repos/CMakeLists.txt b/extensions/rocksdb-repos/CMakeLists.txt
@@ -21,13 +21,8 @@ if (NOT ENABLE_ROCKSDB)
     return()
 endif()
 
-if (BUILD_ROCKSDB)
-    include(BundledRocksDB)
-    use_bundled_rocksdb(${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR})
-else()
-    list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/rocksdb/sys")
-    find_package(RocksDB REQUIRED)
-endif()
+include(GetRocksDB)
+get_rocksdb(${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR})
 
 include(${CMAKE_SOURCE_DIR}/extensions/ExtensionHeader.txt)
 

diff --git a/thirdparty/rocksdb/all/conandata.yml b/thirdparty/rocksdb/all/conandata.yml
@@ -0,0 +1,14 @@
+sources:
+  "8.10.2":
+    url: "https://github.com/facebook/rocksdb/archive/refs/tags/v8.10.2.tar.gz"
+    sha256: "44b6ec2f4723a0d495762da245d4a59d38704e0d9d3d31c45af4014bee853256"
+patches:
+  "8.10.2":
+    - patch_file: "patches/arm7.patch"
+      patch_description: "Fix for ARM7"
+      patch_type: "portability"
+      patch_source: "https://github.com/apache/nifi-minifi-cpp/commit/36244376bb46c5c85657c6dfe92a6a0c4cea650a"
+    - patch_file: "patches/dboptions_equality_operator.patch"
+      patch_description: "Fix for missing equality operator in DBOptions and make DBOptions configurable"
+      patch_type: "portability"
+      patch_source: "https://github.com/apache/nifi-minifi-cpp/commit/545236fd06b613f2d2b38f74ff9ee85df9190d59"