From 71686e39e5922bbcaa48ccb063df1f6ae2d363d8 Mon Sep 17 00:00:00 2001 From: james94 Date: Wed, 2 Oct 2024 00:15:40 -0700 Subject: [PATCH] Created RocksDB Conan Recipe to take patches like BundledRocksDB Also when we create the RocksDB conan package, it not only builds with the patches like BundledRocksDB, it also incorporates ZLib and ZStd conan packages too. Therefore, now that we use a conan RocksDB for building MiNiFi with conan, when we build MiNiFi with standalone CMake, we use the original version of BundledRocksDB that was in 'main' branch before I modified it. I verified that when I build MiNiFi with conan, 218 CTESTs pass out of 219. Similarly, when I build MiNiFi with standalone, then I run CTESTs, 218 out of 219 pass too. --- CMakeLists.txt | 6 +- CONAN.md | 13 + cmake/BundledRocksDB.cmake | 26 +- cmake/GetBZip2.cmake | 27 ++ cmake/GetCatch2.cmake | 2 +- cmake/GetCivetWeb.cmake | 2 +- cmake/GetFmt.cmake | 2 +- cmake/GetLibCURL.cmake | 2 +- cmake/GetLibXml2.cmake | 2 +- cmake/GetOpenSSL.cmake | 2 +- cmake/GetRocksDB.cmake | 34 +++ cmake/GetSpdlog.cmake | 2 +- cmake/GetZLIB.cmake | 2 +- cmake/GetZstd.cmake | 2 +- cmake/MiNiFiOptions.cmake | 2 + conanfile.py | 4 +- extensions/rocksdb-repos/CMakeLists.txt | 9 +- thirdparty/rocksdb/all/conandata.yml | 14 + thirdparty/rocksdb/all/conanfile.py | 243 ++++++++++++++++++ .../rocksdb/{ => all/patches}/arm7.patch | 2 +- .../rocksdb/{ => all/patches}/cstdint.patch | 6 +- .../dboptions_equality_operator.patch | 8 +- thirdparty/rocksdb/config.yml | 3 + 23 files changed, 369 insertions(+), 46 deletions(-) create mode 100644 cmake/GetBZip2.cmake create mode 100644 cmake/GetRocksDB.cmake create mode 100644 thirdparty/rocksdb/all/conandata.yml create mode 100644 thirdparty/rocksdb/all/conanfile.py rename thirdparty/rocksdb/{ => all/patches}/arm7.patch (91%) rename thirdparty/rocksdb/{ => all/patches}/cstdint.patch (97%) rename thirdparty/rocksdb/{ => all/patches}/dboptions_equality_operator.patch (98%) create mode 100644 thirdparty/rocksdb/config.yml diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c460aa0fd..21aa51f223 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -228,8 +228,10 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/ssl") if (ENABLE_BZIP2 AND (ENABLE_LIBARCHIVE OR (ENABLE_ROCKSDB AND NOT WIN32))) - include(BundledBZip2) - use_bundled_bzip2(${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}) + include(GetBZip2) + get_bzip2(${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}) + # include(BundledBZip2) + # use_bundled_bzip2(${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}) list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/bzip2/dummy") endif() diff --git a/CONAN.md b/CONAN.md index adaf10d02d..088f343156 100644 --- a/CONAN.md +++ b/CONAN.md @@ -25,6 +25,7 @@ We will walk through the steps to build MiNiFi using conan version 2 that comes - [Build MiNiFi C++ with Conan](#build-minifi-c---with-conan) - [Create MiNiFi C++ Conan Package](#create-minifi-c---conan-package) - [Conclusion](#conclusion) + - [Appendix](#appendix) ## Description @@ -120,3 +121,15 @@ To have a more consistent quick build process for MiNiFi, we can use conan versi There are multiple benefits of having MiNiFi prebuilt conan packages. We can upload these MiNiFi conan packages to a conan repository like jfrog for version management. We can easily integrate MiNiFi's edge data pipeline features into other C++ software infrastructure using conan's CMake support. We can still use MiNiFi for edge data collection from the IoT devices embedded on robotic systems. We can integrate MiNiFi into self-driving cars (sensor examples: cameras, lidar, radar, inertial measurement unit (IMU), electronic speed controller (ESC), steering servo, etc), into medical imaging robots (sensor examples: depth cameras, ultrasound, gamma detector, force/torque sensor, joint position sensor, etc) or some other real-time robotic system. By leveraging MiNiFi as a conan package, we can leverage MiNiFi that comes with the best practices of building data pipelines from NiFi and bring them into existing C++ real-time robotics infrastructure. Some teams across companies typically have their own custom edge data pipelines that process data for the different events to eventually perform actions on that data. As an alternative to all these companies and their teams having their own custom edge data pipeline libraries, MiNiFi C++, which is like a headless NiFi, can provide a more consistent standard approach for team's to build edge pipelines. Through all stages of the edge data pipelines, MiNiFi can still provide telemetry to NiFi instances running in the cloud. + +## Appendix + +### Create Custom RocksDB Conan Package + +~~~bash +pushd nifi-minifi-cpp/thirdparty/rocksdb/all + +conan create . --user=minifi --channel=develop --version=8.10.2 -pr=../../../etc/conan/profiles/release-linux + +popd +~~~ diff --git a/cmake/BundledRocksDB.cmake b/cmake/BundledRocksDB.cmake index 34882f26de..ac2a6e203a 100644 --- a/cmake/BundledRocksDB.cmake +++ b/cmake/BundledRocksDB.cmake @@ -19,8 +19,7 @@ function(use_bundled_rocksdb SOURCE_DIR BINARY_DIR) message("Using bundled RocksDB") if (NOT WIN32) - include(GetZstd) - get_zstd() + include(Zstd) list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/zstd/dummy") include(LZ4) @@ -28,9 +27,9 @@ function(use_bundled_rocksdb SOURCE_DIR BINARY_DIR) endif() # Patch to fix build issue on ARM7 architecture: https://github.com/facebook/rocksdb/issues/8609#issuecomment-1009572506 - set(PATCH_FILE_1 "${SOURCE_DIR}/thirdparty/rocksdb/arm7.patch") - set(PATCH_FILE_2 "${SOURCE_DIR}/thirdparty/rocksdb/dboptions_equality_operator.patch") - set(PATCH_FILE_3 "${SOURCE_DIR}/thirdparty/rocksdb/cstdint.patch") + set(PATCH_FILE_1 "${SOURCE_DIR}/thirdparty/rocksdb/all/patches/arm7.patch") + set(PATCH_FILE_2 "${SOURCE_DIR}/thirdparty/rocksdb/all/patches/dboptions_equality_operator.patch") + set(PATCH_FILE_3 "${SOURCE_DIR}/thirdparty/rocksdb/all/patches/cstdint.patch") set(PC ${Bash_EXECUTABLE} -c "set -x &&\ (\"${Patch_EXECUTABLE}\" -p1 -R -s -f --dry-run -i \"${PATCH_FILE_1}\" || \"${Patch_EXECUTABLE}\" -p1 -N -i \"${PATCH_FILE_1}\") &&\ (\"${Patch_EXECUTABLE}\" -p1 -R -s -f --dry-run -i \"${PATCH_FILE_2}\" || \"${Patch_EXECUTABLE}\" -p1 -N -i \"${PATCH_FILE_2}\") &&\ @@ -64,17 +63,10 @@ function(use_bundled_rocksdb SOURCE_DIR BINARY_DIR) -DROCKSDB_INSTALL_ON_WINDOWS=ON -DWITH_XPRESS=ON) else() - if(MINIFI_ZSTD_SOURCE STREQUAL "CONAN" AND MINIFI_ZLIB_SOURCE STREQUAL "CONAN") - list(APPEND ROCKSDB_CMAKE_ARGS - -DWITH_ZLIB=OFF - -DWITH_ZSTD=OFF) - else() - list(APPEND ROCKSDB_CMAKE_ARGS - -DWITH_ZLIB=ON - -DWITH_ZSTD=ON) - endif() list(APPEND ROCKSDB_CMAKE_ARGS + -DWITH_ZLIB=ON -DWITH_BZ2=ON + -DWITH_ZSTD=ON -DWITH_LZ4=ON) endif() @@ -103,11 +95,7 @@ function(use_bundled_rocksdb SOURCE_DIR BINARY_DIR) add_library(RocksDB::RocksDB STATIC IMPORTED) set_target_properties(RocksDB::RocksDB PROPERTIES IMPORTED_LOCATION "${ROCKSDB_LIBRARY}") if (NOT WIN32) - if(MINIFI_ZSTD_SOURCE STREQUAL "CONAN" AND MINIFI_ZLIB_SOURCE STREQUAL "CONAN") - add_dependencies(rocksdb-external BZip2::BZip2 lz4::lz4) - else() - add_dependencies(rocksdb-external ZLIB::ZLIB BZip2::BZip2 zstd::zstd lz4::lz4) - endif() + add_dependencies(rocksdb-external ZLIB::ZLIB BZip2::BZip2 zstd::zstd lz4::lz4) endif() add_dependencies(RocksDB::RocksDB rocksdb-external) file(MAKE_DIRECTORY ${ROCKSDB_INCLUDE_DIR}) diff --git a/cmake/GetBZip2.cmake b/cmake/GetBZip2.cmake new file mode 100644 index 0000000000..be84ec1313 --- /dev/null +++ b/cmake/GetBZip2.cmake @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function(get_bzip2 SOURCE_DIR BINARY_DIR) + if(MINIFI_BZIP2_SOURCE STREQUAL "CONAN") + message("Using Conan Packager to manage installing prebuilt bzip2 external lib") + find_package(BZip2 REQUIRED) + elseif(MINIFI_BZIP2_SOURCE STREQUAL "BUILD") + message("Using CMake to build bzip2 from source") + include(BundledBZip2) + use_bundled_bzip2(${SOURCE_DIR} ${BINARY_DIR}) + endif() +endfunction(get_bzip2) diff --git a/cmake/GetCatch2.cmake b/cmake/GetCatch2.cmake index 23ff9dbee7..b9d652a4ee 100644 --- a/cmake/GetCatch2.cmake +++ b/cmake/GetCatch2.cmake @@ -21,7 +21,7 @@ function(get_catch2) find_package(Catch2 REQUIRED) add_library(Catch2WithMain ALIAS Catch2::Catch2WithMain) elseif(MINIFI_CATCH2_SOURCE STREQUAL "BUILD") - message("Using CMAKE's ExternalProject_Add to manage source building Catch2 external lib") + message("Using CMake to build Catch2 from source") include(Catch2) endif() endfunction(get_catch2) diff --git a/cmake/GetCivetWeb.cmake b/cmake/GetCivetWeb.cmake index 3e03b4fbc5..0657e03836 100644 --- a/cmake/GetCivetWeb.cmake +++ b/cmake/GetCivetWeb.cmake @@ -20,7 +20,7 @@ function(get_civetweb) message("Using Conan Packager to manage installing prebuilt CivetWeb external lib") find_package(civetweb REQUIRED) elseif(MINIFI_CIVETWEB_SOURCE STREQUAL "BUILD") - message("Using CMAKE's ExternalProject_Add to manage source building CivetWeb external lib") + message("Using CMake to build CivetWeb from source") include(CivetWeb) endif() endfunction(get_civetweb) diff --git a/cmake/GetFmt.cmake b/cmake/GetFmt.cmake index 94bce3c218..25b15aaf1b 100644 --- a/cmake/GetFmt.cmake +++ b/cmake/GetFmt.cmake @@ -20,7 +20,7 @@ function(get_fmt) message("Using Conan Packager to manage installing prebuilt Fmt external lib") find_package(fmt REQUIRED) elseif(MINIFI_FMT_SOURCE STREQUAL "BUILD") - message("Using CMAKE's ExternalProject_Add to manage source building Fmt external lib") + message("Using CMake to build Fmt from source") include(fmt) endif() endfunction(get_fmt) diff --git a/cmake/GetLibCURL.cmake b/cmake/GetLibCURL.cmake index b2dff1892c..86f8a87f75 100644 --- a/cmake/GetLibCURL.cmake +++ b/cmake/GetLibCURL.cmake @@ -20,7 +20,7 @@ function(get_curl SOURCE_DIR BINARY_DIR) message("Using Conan Packager to manage installing prebuilt libcurl external lib") find_package(CURL REQUIRED) elseif(MINIFI_LIBCURL_SOURCE STREQUAL "BUILD") - message("Using CMAKE's ExternalProject_Add to manage source building libcurl external lib") + message("Using CMake to build libcurl from source") include(BundledLibcURL) use_bundled_curl(${SOURCE_DIR} ${BINARY_DIR}) endif() diff --git a/cmake/GetLibXml2.cmake b/cmake/GetLibXml2.cmake index 369f1285c9..4e780e29c8 100644 --- a/cmake/GetLibXml2.cmake +++ b/cmake/GetLibXml2.cmake @@ -20,7 +20,7 @@ function(get_libxml2 SOURCE_DIR BINARY_DIR) message("Using Conan Packager to manage installing prebuilt LibXml2 external lib") find_package(libxml2 REQUIRED) elseif(MINIFI_LIBXML2_SOURCE STREQUAL "BUILD") - message("Using CMAKE's ExternalProject_Add to manage source building LibXml2 external lib") + message("Using CMake to build LibXml2 from source") include(BundledLibXml2) use_bundled_libxml2(${SOURCE_DIR} ${BINARY_DIR}) endif() diff --git a/cmake/GetOpenSSL.cmake b/cmake/GetOpenSSL.cmake index 21d34dadb9..3029d7e1f0 100644 --- a/cmake/GetOpenSSL.cmake +++ b/cmake/GetOpenSSL.cmake @@ -20,7 +20,7 @@ function(get_openssl SOURCE_DIR BINARY_DIR) message("Using Conan Packager to manage installing prebuilt OpenSSL external lib") find_package(OpenSSL REQUIRED) elseif(MINIFI_OPENSSL_SOURCE STREQUAL "BUILD") - message("Using CMAKE's ExternalProject_Add to manage source building OpenSSL external lib") + message("Using CMake to build OpenSSL from source") include(BundledOpenSSL) use_openssl(${SOURCE_DIR} ${BINARY_DIR}) endif() diff --git a/cmake/GetRocksDB.cmake b/cmake/GetRocksDB.cmake new file mode 100644 index 0000000000..095ae97386 --- /dev/null +++ b/cmake/GetRocksDB.cmake @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function(get_rocksdb SOURCE_DIR BINARY_DIR) + if(MINIFI_ROCKSDB_SOURCE STREQUAL "CONAN") + message("Using Conan Packager to manage installing prebuilt RocksDB external lib") + find_package(RocksDB REQUIRED) + add_library(RocksDB::RocksDB ALIAS RocksDB::rocksdb) + elseif(MINIFI_ROCKSDB_SOURCE STREQUAL "BUILD") + message("Using CMake to build RocksDB from source") + + if (BUILD_ROCKSDB) + include(BundledRocksDB) + use_bundled_rocksdb(${SOURCE_DIR} ${BINARY_DIR}) + else() + list(APPEND CMAKE_MODULE_PATH "${SOURCE_DIR}/cmake/rocksdb/sys") + find_package(RocksDB REQUIRED) + endif() + endif() +endfunction(get_rocksdb SOURCE_DIR BINARY_DIR) diff --git a/cmake/GetSpdlog.cmake b/cmake/GetSpdlog.cmake index b9f4233eff..b8b05f9992 100644 --- a/cmake/GetSpdlog.cmake +++ b/cmake/GetSpdlog.cmake @@ -25,7 +25,7 @@ function(get_spdlog) add_library(spdlog ALIAS spdlog::spdlog) elseif(MINIFI_SPDLOG_SOURCE STREQUAL "BUILD") - message("Using CMAKE's ExternalProject_Add to manage source building Spdlog external lib") + message("Using CMake to build Spdlog from source") include(Spdlog) endif() endfunction(get_spdlog) diff --git a/cmake/GetZLIB.cmake b/cmake/GetZLIB.cmake index 9b018df76e..8111ca631a 100644 --- a/cmake/GetZLIB.cmake +++ b/cmake/GetZLIB.cmake @@ -20,7 +20,7 @@ function(get_zlib SOURCE_DIR BINARY_DIR) message("Using Conan Packager to manage installing prebuilt zlib external lib") find_package(ZLIB REQUIRED) elseif(MINIFI_ZLIB_SOURCE STREQUAL "BUILD") - message("Using CMAKE's ExternalProject_Add to manage source building zlib external lib") + message("Using CMake to build zlib from source") include(BundledZLIB) use_bundled_zlib(${SOURCE_DIR} ${BINARY_DIR}) endif() diff --git a/cmake/GetZstd.cmake b/cmake/GetZstd.cmake index 2b23929531..596df4aaef 100644 --- a/cmake/GetZstd.cmake +++ b/cmake/GetZstd.cmake @@ -21,7 +21,7 @@ function(get_zstd) find_package(zstd REQUIRED) add_library(zstd::zstd ALIAS zstd::libzstd_static) elseif(MINIFI_ZSTD_SOURCE STREQUAL "BUILD") - message("Using CMAKE's ExternalProject_Add to manage source building zstd external lib") + message("Using CMake to build zstd from source") include(Zstd) endif() endfunction(get_zstd) diff --git a/cmake/MiNiFiOptions.cmake b/cmake/MiNiFiOptions.cmake index dac88ce852..3546362c8b 100644 --- a/cmake/MiNiFiOptions.cmake +++ b/cmake/MiNiFiOptions.cmake @@ -139,7 +139,9 @@ set_property(CACHE STRICT_GSL_CHECKS PROPERTY STRINGS ${STRICT_GSL_CHECKS_Values add_minifi_multi_option(MINIFI_LIBCURL_SOURCE "Retrieves LibCURL from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_OPENSSL_SOURCE "Retrieves OpenSSL from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_ZLIB_SOURCE "Retrieves ZLib from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_ROCKSDB_SOURCE "Retrieves RocksDB from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_ZSTD_SOURCE "Retrieves Zstd from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_BZIP2_SOURCE "Retrieves BZip2 from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_CIVETWEB_SOURCE "Retrieves CivetWeb from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_LIBXML2_SOURCE "Retrieves LibXml2 from provided source" "BUILD;SYSTEM;CONAN" "BUILD") add_minifi_multi_option(MINIFI_FMT_SOURCE "Retrieves Fmt from provided source" "BUILD;SYSTEM;CONAN" "BUILD") diff --git a/conanfile.py b/conanfile.py index 0f4f82b27b..5dd4db1fca 100644 --- a/conanfile.py +++ b/conanfile.py @@ -7,7 +7,7 @@ required_conan_version = ">=2.0" -shared_requires = ("openssl/3.2.1", "libcurl/8.4.0", "civetweb/1.16", "libxml2/2.12.6", "fmt/10.2.1", "spdlog/1.14.0", "catch2/3.5.4", "zlib/1.2.11", "zstd/1.5.2") +shared_requires = ("openssl/3.2.1", "libcurl/8.4.0", "civetweb/1.16", "libxml2/2.12.6", "fmt/10.2.1", "spdlog/1.14.0", "catch2/3.5.4", "zlib/1.2.11", "zstd/1.5.2", "bzip2/1.0.8", "rocksdb/8.10.2@minifi/develop") shared_sources = ("CMakeLists.txt", "libminifi/*", "extensions/*", "minifi_main/*", "nanofi/*", "bin/*", "bootstrap/*", "cmake/*", "conf/*", "controller/*", "encrypt-config/*", "etc/*", "examples/*", "msi/*", "thirdparty/*", "docker/*", "LICENSE", "NOTICE", "README.md", "C2.md", "CONFIGURE.md", "CONTRIBUTING.md", "CONTROLLERS.md", "EXPRESSIONS.md", "Extensions.md", "JNI.md", "METRICS.md", "OPS.md", "PROCESSORS.md", "ThirdParties.md", "Windows.md", "aptitude.sh", "arch.sh", "bootstrap.sh", "bstrp_functions.sh", "centos.sh", "CPPLINT.cfg", "darwin.sh", "debian.sh", "deploy.sh", "fedora.sh", "generateVersion.sh", "linux.sh", "rheldistro.sh", "run_clang_tidy.sh", "run_clang_tidy.sh", "run_flake8.sh", "run_shellcheck.sh", "suse.sh", "versioninfo.rc.in") @@ -31,7 +31,9 @@ def generate(self): tc.variables["MINIFI_LIBCURL_SOURCE"] = "CONAN" tc.variables["MINIFI_OPENSSL_SOURCE"] = "CONAN" tc.variables["MINIFI_ZLIB_SOURCE"] = "CONAN" + tc.variables["MINIFI_ROCKSDB_SOURCE"] = "CONAN" tc.variables["MINIFI_ZSTD_SOURCE"] = "CONAN" + tc.variables["MINIFI_BZIP2_SOURCE"] = "CONAN" tc.variables["MINIFI_CIVETWEB_SOURCE"] = "CONAN" tc.variables["MINIFI_LIBXML2_SOURCE"] = "CONAN" tc.variables["MINIFI_FMT_SOURCE"] = "CONAN" diff --git a/extensions/rocksdb-repos/CMakeLists.txt b/extensions/rocksdb-repos/CMakeLists.txt index e9d0a7af8d..2f43753c7e 100644 --- a/extensions/rocksdb-repos/CMakeLists.txt +++ b/extensions/rocksdb-repos/CMakeLists.txt @@ -21,13 +21,8 @@ if (NOT ENABLE_ROCKSDB) return() endif() -if (BUILD_ROCKSDB) - include(BundledRocksDB) - use_bundled_rocksdb(${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}) -else() - list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/rocksdb/sys") - find_package(RocksDB REQUIRED) -endif() +include(GetRocksDB) +get_rocksdb(${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}) include(${CMAKE_SOURCE_DIR}/extensions/ExtensionHeader.txt) diff --git a/thirdparty/rocksdb/all/conandata.yml b/thirdparty/rocksdb/all/conandata.yml new file mode 100644 index 0000000000..ed09c733cf --- /dev/null +++ b/thirdparty/rocksdb/all/conandata.yml @@ -0,0 +1,14 @@ +sources: + "8.10.2": + url: "https://github.com/facebook/rocksdb/archive/refs/tags/v8.10.2.tar.gz" + sha256: "44b6ec2f4723a0d495762da245d4a59d38704e0d9d3d31c45af4014bee853256" +patches: + "8.10.2": + - patch_file: "patches/arm7.patch" + patch_description: "Fix for ARM7" + patch_type: "portability" + patch_source: "https://github.com/apache/nifi-minifi-cpp/commit/36244376bb46c5c85657c6dfe92a6a0c4cea650a" + - patch_file: "patches/dboptions_equality_operator.patch" + patch_description: "Fix for missing equality operator in DBOptions and make DBOptions configurable" + patch_type: "portability" + patch_source: "https://github.com/apache/nifi-minifi-cpp/commit/545236fd06b613f2d2b38f74ff9ee85df9190d59" diff --git a/thirdparty/rocksdb/all/conanfile.py b/thirdparty/rocksdb/all/conanfile.py new file mode 100644 index 0000000000..d9a5ae5627 --- /dev/null +++ b/thirdparty/rocksdb/all/conanfile.py @@ -0,0 +1,243 @@ +import os +import glob +import shutil + +from conan import ConanFile +from conan.errors import ConanInvalidConfiguration +from conan.tools.build import check_min_cppstd +from conan.tools.cmake import CMake, CMakeDeps, CMakeToolchain, cmake_layout +from conan.tools.files import apply_conandata_patches, collect_libs, copy, export_conandata_patches, get, rm, rmdir +from conan.tools.microsoft import check_min_vs, is_msvc, is_msvc_static_runtime +from conan.tools.scm import Version + +required_conan_version = ">=1.53.0" + + +class RocksDBConan(ConanFile): + name = "rocksdb" + homepage = "https://github.com/facebook/rocksdb" + license = ("GPL-2.0-only", "Apache-2.0") + url = "https://github.com/conan-io/conan-center-index" + description = "A library that provides an embeddable, persistent key-value store for fast storage" + topics = ("database", "leveldb", "facebook", "key-value") + package_type = "library" + settings = "os", "arch", "compiler", "build_type" + options = { + "shared": [True, False], + "fPIC": [True, False], + "lite": [True, False], + "with_gflags": [True, False], + "with_snappy": [True, False], + "with_lz4": [True, False], + "with_zlib": [True, False], + "with_bz2": [True, False], + "with_zstd": [True, False], + "with_tbb": [True, False], + "with_jemalloc": [True, False], + "enable_sse": [False, "sse42", "avx2"], + "use_rtti": [True, False], + } + default_options = { + "shared": False, + "fPIC": True, + "lite": False, + "with_snappy": False, + "with_lz4": True, + "with_zlib": True, + "with_bz2": True, + "with_zstd": True, + "with_gflags": False, + "with_tbb": False, + "with_jemalloc": False, + "enable_sse": False, + "use_rtti": True, + } + + @property + def _min_cppstd(self): + return "11" if Version(self.version) < "8.8.1" else "20" + + @property + def _compilers_minimum_version(self): + return {} if self._min_cppstd == "11" else { + "apple-clang": "10", + "clang": "7", + "gcc": "7", + "msvc": "191", + "Visual Studio": "15", + } + + def export_sources(self): + export_conandata_patches(self) + + def config_options(self): + if self.settings.os == "Windows": + del self.options.fPIC + if self.settings.arch != "x86_64": + del self.options.with_tbb + if self.settings.build_type == "Debug": + self.options.use_rtti = True # Rtti are used in asserts for debug mode... + + def configure(self): + if self.options.shared: + self.options.rm_safe("fPIC") + + def layout(self): + cmake_layout(self, src_folder="src") + + def requirements(self): + if self.options.with_gflags: + self.requires("gflags/2.2.2") + if self.options.with_snappy: + self.requires("snappy/1.1.10") + if self.options.with_lz4: + self.requires("lz4/1.9.4") + if self.options.with_zlib: + self.requires("zlib/[>=1.2.11 <2]") + if self.options.with_bz2: + self.requires("bzip2/1.0.8") + # self.requires("bzip2/1.0.8@minifi/dev") # prebuilt with minifi bz2 patch + if self.options.with_zstd: + self.requires("zstd/1.5.2") + if self.options.get_safe("with_tbb"): + self.requires("onetbb/2021.10.0") + if self.options.with_jemalloc: + self.requires("jemalloc/5.3.0") + + def validate(self): + if self.settings.compiler.get_safe("cppstd"): + check_min_cppstd(self, self._min_cppstd) + + minimum_version = self._compilers_minimum_version.get(str(self.settings.compiler), False) + if minimum_version and Version(self.settings.compiler.version) < minimum_version: + raise ConanInvalidConfiguration( + f"{self.ref} requires C++{self._min_cppstd}, which your compiler does not support." + ) + + if self.settings.arch not in ["x86_64", "ppc64le", "ppc64", "mips64", "armv8"]: + raise ConanInvalidConfiguration("Rocksdb requires 64 bits") + + check_min_vs(self, "191") + + if self.version == "6.20.3" and \ + self.settings.os == "Linux" and \ + self.settings.compiler == "gcc" and \ + Version(self.settings.compiler.version) < "5": + raise ConanInvalidConfiguration("Rocksdb 6.20.3 is not compilable with gcc <5.") # See https://github.com/facebook/rocksdb/issues/3522 + + def source(self): + get(self, **self.conan_data["sources"][self.version], strip_root=True) + + def generate(self): + tc = CMakeToolchain(self) + tc.variables["FAIL_ON_WARNINGS"] = False + tc.variables["WITH_TESTS"] = False + tc.variables["WITH_TOOLS"] = True + tc.variables["WITH_CORE_TOOLS"] = False + tc.variables["WITH_BENCHMARK_TOOLS"] = False + tc.variables["WITH_FOLLY_DISTRIBUTED_MUTEX"] = False + if is_msvc(self): + tc.variables["WITH_MD_LIBRARY"] = not is_msvc_static_runtime(self) + tc.variables["ROCKSDB_INSTALL_ON_WINDOWS"] = self.settings.os == "Windows" + tc.variables["ROCKSDB_LITE"] = self.options.lite + tc.variables["WITH_GFLAGS"] = self.options.with_gflags + tc.variables["WITH_SNAPPY"] = self.options.with_snappy + tc.variables["WITH_LZ4"] = self.options.with_lz4 + tc.variables["WITH_ZLIB"] = self.options.with_zlib + tc.variables["WITH_BZ2"] = self.options.with_bz2 + tc.variables["WITH_ZSTD"] = self.options.with_zstd + tc.variables["WITH_TBB"] = self.options.get_safe("with_tbb", False) + tc.variables["WITH_JEMALLOC"] = self.options.with_jemalloc + tc.variables["ROCKSDB_BUILD_SHARED"] = self.options.shared + tc.variables["ROCKSDB_LIBRARY_EXPORTS"] = self.settings.os == "Windows" and self.options.shared + tc.variables["ROCKSDB_DLL" ] = self.settings.os == "Windows" and self.options.shared + tc.variables["USE_RTTI"] = self.options.use_rtti + if not bool(self.options.enable_sse): + tc.variables["PORTABLE"] = True + tc.variables["FORCE_SSE42"] = False + elif self.options.enable_sse == "sse42": + tc.variables["PORTABLE"] = True + tc.variables["FORCE_SSE42"] = True + elif self.options.enable_sse == "avx2": + tc.variables["PORTABLE"] = True + tc.variables["FORCE_SSE42"] = False + # not available yet in CCI + tc.variables["WITH_NUMA"] = False + tc.generate() + + deps = CMakeDeps(self) + if self.options.with_jemalloc: + deps.set_property("jemalloc", "cmake_file_name", "JeMalloc") + deps.set_property("jemalloc", "cmake_target_name", "JeMalloc::JeMalloc") + if self.options.with_zstd: + deps.set_property("zstd", "cmake_target_name", "zstd::zstd") + deps.generate() + + def build(self): + apply_conandata_patches(self) + cmake = CMake(self) + cmake.configure() + cmake.build() + + def _remove_static_libraries(self): + rm(self, "rocksdb.lib", os.path.join(self.package_folder, "lib")) + for lib in glob.glob(os.path.join(self.package_folder, "lib", "*.a")): + if not lib.endswith(".dll.a"): + os.remove(lib) + + def _remove_cpp_headers(self): + for path in glob.glob(os.path.join(self.package_folder, "include", "rocksdb", "*")): + if path != os.path.join(self.package_folder, "include", "rocksdb", "c.h"): + if os.path.isfile(path): + os.remove(path) + else: + shutil.rmtree(path) + + def package(self): + copy(self, "COPYING", src=self.source_folder, dst=os.path.join(self.package_folder, "licenses")) + copy(self, "LICENSE*", src=self.source_folder, dst=os.path.join(self.package_folder, "licenses")) + cmake = CMake(self) + cmake.install() + if self.options.shared: + self._remove_static_libraries() + self._remove_cpp_headers() # Force stable ABI for shared libraries + rmdir(self, os.path.join(self.package_folder, "lib", "cmake")) + rmdir(self, os.path.join(self.package_folder, "lib", "pkgconfig")) + + def package_info(self): + cmake_target = "rocksdb-shared" if self.options.shared else "rocksdb" + self.cpp_info.set_property("cmake_file_name", "RocksDB") + self.cpp_info.set_property("cmake_target_name", f"RocksDB::{cmake_target}") + # TODO: back to global scope in conan v2 once cmake_find_package* generators removed + self.cpp_info.components["librocksdb"].libs = collect_libs(self) + if self.settings.os == "Windows": + self.cpp_info.components["librocksdb"].system_libs = ["shlwapi", "rpcrt4"] + if self.options.shared: + self.cpp_info.components["librocksdb"].defines = ["ROCKSDB_DLL"] + elif self.settings.os in ["Linux", "FreeBSD"]: + self.cpp_info.components["librocksdb"].system_libs = ["pthread", "m"] + if self.options.lite: + self.cpp_info.components["librocksdb"].defines.append("ROCKSDB_LITE") + + # TODO: to remove in conan v2 once cmake_find_package* generators removed + self.cpp_info.names["cmake_find_package"] = "RocksDB" + self.cpp_info.names["cmake_find_package_multi"] = "RocksDB" + self.cpp_info.components["librocksdb"].names["cmake_find_package"] = cmake_target + self.cpp_info.components["librocksdb"].names["cmake_find_package_multi"] = cmake_target + self.cpp_info.components["librocksdb"].set_property("cmake_target_name", f"RocksDB::{cmake_target}") + if self.options.with_gflags: + self.cpp_info.components["librocksdb"].requires.append("gflags::gflags") + if self.options.with_snappy: + self.cpp_info.components["librocksdb"].requires.append("snappy::snappy") + if self.options.with_lz4: + self.cpp_info.components["librocksdb"].requires.append("lz4::lz4") + if self.options.with_zlib: + self.cpp_info.components["librocksdb"].requires.append("zlib::zlib") + if self.options.with_bz2: + self.cpp_info.components["librocksdb"].requires.append("bzip2::bzip2") + if self.options.with_zstd: + self.cpp_info.components["librocksdb"].requires.append("zstd::zstd") + if self.options.get_safe("with_tbb"): + self.cpp_info.components["librocksdb"].requires.append("onetbb::onetbb") + if self.options.with_jemalloc: + self.cpp_info.components["librocksdb"].requires.append("jemalloc::jemalloc") diff --git a/thirdparty/rocksdb/arm7.patch b/thirdparty/rocksdb/all/patches/arm7.patch similarity index 91% rename from thirdparty/rocksdb/arm7.patch rename to thirdparty/rocksdb/all/patches/arm7.patch index 1eb64ed5ce..bd1eb6aa22 100644 --- a/thirdparty/rocksdb/arm7.patch +++ b/thirdparty/rocksdb/all/patches/arm7.patch @@ -4,7 +4,7 @@ index 225e3fa72..cd5f935f1 100644 +++ b/utilities/transactions/lock/range/range_tree/lib/portability/toku_time.h @@ -131,6 +131,10 @@ static inline tokutime_t toku_time_now(void) { uint64_t result; - __asm __volatile__("mrs %[rt], cntvct_el0" : [ rt ] "=r"(result)); + __asm __volatile__("mrs %[rt], cntvct_el0" : [rt] "=r"(result)); return result; +#elif defined(__arm__) + uint32_t lo, hi; diff --git a/thirdparty/rocksdb/cstdint.patch b/thirdparty/rocksdb/all/patches/cstdint.patch similarity index 97% rename from thirdparty/rocksdb/cstdint.patch rename to thirdparty/rocksdb/all/patches/cstdint.patch index 2435905c9a..4cff815831 100644 --- a/thirdparty/rocksdb/cstdint.patch +++ b/thirdparty/rocksdb/all/patches/cstdint.patch @@ -3,10 +3,10 @@ index 75d61abb4..f42ef6dc2 100644 --- a/options/offpeak_time_info.h +++ b/options/offpeak_time_info.h @@ -5,6 +5,7 @@ - + #pragma once - + +#include #include - + #include "rocksdb/rocksdb_namespace.h" diff --git a/thirdparty/rocksdb/dboptions_equality_operator.patch b/thirdparty/rocksdb/all/patches/dboptions_equality_operator.patch similarity index 98% rename from thirdparty/rocksdb/dboptions_equality_operator.patch rename to thirdparty/rocksdb/all/patches/dboptions_equality_operator.patch index 2efb91ea0e..cff3dd31ea 100644 --- a/thirdparty/rocksdb/dboptions_equality_operator.patch +++ b/thirdparty/rocksdb/all/patches/dboptions_equality_operator.patch @@ -3,19 +3,19 @@ index ae5ed2c26..0038c6bff 100644 --- a/include/rocksdb/options.h +++ b/include/rocksdb/options.h @@ -397,6 +397,7 @@ struct DbPath { - + DbPath() : target_size(0) {} DbPath(const std::string& p, uint64_t t) : path(p), target_size(t) {} + bool operator==(const DbPath& other) const = default; }; - + extern const char* kHostnameForDbHostId; @@ -1008,6 +1009,8 @@ struct DBOptions { // Create DBOptions from Options explicit DBOptions(const Options& options); - + + bool operator==(const DBOptions& other) const = default; + void Dump(Logger* log) const; - + // Allows OS to incrementally sync files to disk while they are being diff --git a/thirdparty/rocksdb/config.yml b/thirdparty/rocksdb/config.yml new file mode 100644 index 0000000000..21a3c25eab --- /dev/null +++ b/thirdparty/rocksdb/config.yml @@ -0,0 +1,3 @@ +versions: + "8.10.2": + folder: all