diff --git a/.github/workflows/cpp-ci.yml b/.github/workflows/cpp-ci.yml index ab5e863..84fe572 100644 --- a/.github/workflows/cpp-ci.yml +++ b/.github/workflows/cpp-ci.yml @@ -18,19 +18,24 @@ jobs: - name: Install dependencies uses: aminya/setup-cpp@v1 with: - conan: 1.58.0 + # conan: 1.58.0 cmake: true - - name: setup conan - run: - conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install libcurl4-openssl-dev + + # - name: setup conan + # run: + # conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local - - name: conan package cache - uses: actions/cache@v3 - with: - path: ~/.conan - key: conan-${{ hashFiles('./cpp/conanfile.py') }} - restore-keys: conan- + # - name: conan package cache + # uses: actions/cache@v3 + # with: + # path: ~/.conan + # key: conan-${{ hashFiles('./cpp/conanfile.py') }} + # restore-keys: conan- - name: Build working-directory: ./cpp @@ -52,7 +57,7 @@ jobs: make check-tidy - name : Test - working-directory: ./cpp/build/Release + working-directory: ./cpp/build/test run: ctest . diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 8111a2a..b8b9c96 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -7,28 +7,33 @@ option(WITH_ASAN "Build with address sanitizer." OFF) option(WITH_OPENDAL "Build with opendal." OFF) set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") if (WITH_OPENDAL) add_compile_definitions(MILVUS_OPENDAL) - list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") include(libopendal) endif() -find_package(Boost REQUIRED) -find_package(Arrow REQUIRED) -find_package(protobuf REQUIRED) -find_package(glog REQUIRED) +include(libarrow) +include(libglog) +add_subdirectory(thirdparty) +find_package(Protobuf REQUIRED) +include_directories(${protobuf_INCLUDE_DIR}) file(GLOB_RECURSE SRC_FILES src/*.cpp src/*.cc) add_library(milvus-storage ${SRC_FILES}) -target_include_directories(milvus-storage PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/milvus-storage ${CMAKE_CURRENT_SOURCE_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR}/test/include) +target_include_directories(milvus-storage PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/milvus-storage ${CMAKE_CURRENT_SOURCE_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR}/test/include ) set(LINK_LIBS - arrow::libarrow - arrow::libparquet - Boost::boost - protobuf::protobuf - glog::glog) + libarrow + libparquet + Boost::uuid + Boost::algorithm + protobuf + Boost::filesystem + libglog + ) if (WITH_OPENDAL) list(APPEND LINK_LIBS opendal) @@ -37,6 +42,5 @@ endif() target_link_libraries(milvus-storage PUBLIC ${LINK_LIBS}) if (WITH_UT) - enable_testing() add_subdirectory(test) endif() diff --git a/cpp/Makefile b/cpp/Makefile index 39818f6..f7f079e 100644 --- a/cpp/Makefile +++ b/cpp/Makefile @@ -7,19 +7,19 @@ endif build: mkdir -p build && cd build && \ - conan install .. --build=missing && \ - conan build .. + cmake .. && \ + make -j 8 debug: mkdir -p build && cd build && \ - conan install .. --build=missing -s build_type=Debug && \ - conan build .. + cmake -DCMAKE_BUILD_TYPE=Debug .. && \ + make -j 8 clean: rm -rf build test: build - cd build/Release && ctest . -j ${TEST_THREADS} --output-on-failure + cd build/test && ctest . -j ${TEST_THREADS} --output-on-failure fix-format: find ./src -type f ! -name "*.pb.h" -iname *.h -o -iname *.cpp | xargs clang-format -i @@ -32,10 +32,10 @@ check-format: find ./test -type f ! -name "*.pb.h" -iname *.h -o -iname *.cpp | xargs clang-format --dry-run --Werror check-tidy: - python3 ./scripts/run-clang-tidy.py -p build/Release + python3 ./scripts/run-clang-tidy.py -p build fix-tidy: - python3 ./scripts/run-clang-tidy.py -fix -p build/Release + python3 ./scripts/run-clang-tidy.py -fix -p build proto: protoc -I="src/proto" --cpp_out="src/proto" src/proto/*.proto diff --git a/cpp/cmake/libarrow.cmake b/cpp/cmake/libarrow.cmake new file mode 100644 index 0000000..640bd94 --- /dev/null +++ b/cpp/cmake/libarrow.cmake @@ -0,0 +1,46 @@ +function(build_arrow) + include(ExternalProject) + set(ARROW_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/thirdparty/arrow-ep) + + file(MAKE_DIRECTORY + ${ARROW_PREFIX} + "${ARROW_PREFIX}/include" + "${ARROW_PREFIX}/lib" + ) + ExternalProject_Add( + arrow_ep + GIT_REPOSITORY https://github.com/apache/arrow.git + GIT_TAG apache-arrow-16.0.0 + CMAKE_ARGS + -DARROW_PARQUET=ON + -DARROW_FILESYSTEM=ON + -DARROW_S3=ON + -DCMAKE_INSTALL_PREFIX:PATH= + SOURCE_SUBDIR cpp + INSTALL_DIR ${ARROW_PREFIX} + ) + + ExternalProject_Get_Property(arrow_ep install_dir) + + message(STATUS ${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/release/libparquet.so) + add_library(libarrow SHARED IMPORTED) + set_target_properties(libarrow + PROPERTIES + IMPORTED_GLOBAL TRUE + IMPORTED_LOCATION ${ARROW_PREFIX}/lib/libarrow.so + INTERFACE_INCLUDE_DIRECTORIES ${install_dir}/include/ + ) + + add_library(libparquet SHARED IMPORTED) + set_target_properties(libparquet + PROPERTIES + IMPORTED_GLOBAL TRUE + IMPORTED_LOCATION ${ARROW_PREFIX}/lib/libparquet.so + INTERFACE_INCLUDE_DIRECTORIES ${install_dir}/include/ + ) + + add_dependencies(libarrow arrow_ep) + add_dependencies(libparquet arrow_ep) +endfunction() + +build_arrow() diff --git a/cpp/cmake/libglog.cmake b/cpp/cmake/libglog.cmake new file mode 100644 index 0000000..cb0776a --- /dev/null +++ b/cpp/cmake/libglog.cmake @@ -0,0 +1,32 @@ +function(build_glog) + include(ExternalProject) + set(GLOG_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/thirdparty/glog-ep) + + file(MAKE_DIRECTORY + ${GLOG_PREFIX} + "${GLOG_PREFIX}/include" + "${GLOG_PREFIX}/lib" + ) + ExternalProject_Add( + glog_ep + GIT_REPOSITORY https://github.com/google/glog.git + GIT_TAG v0.6.0 + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX:PATH= + INSTALL_DIR ${GLOG_PREFIX} + ) + + ExternalProject_Get_Property(glog_ep install_dir) + + add_library(libglog SHARED IMPORTED) + set_target_properties(libglog + PROPERTIES + IMPORTED_GLOBAL TRUE + IMPORTED_LOCATION ${GLOG_PREFIX}/lib/libglog.so + INTERFACE_INCLUDE_DIRECTORIES ${install_dir}/include/ + ) + + add_dependencies(libglog glog_ep) +endfunction() + +build_glog() \ No newline at end of file diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 5ad9cb2..b4c8192 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -1,14 +1,14 @@ -find_package(GTest REQUIRED) +enable_testing() -file(GLOB_RECURSE BUSTUB_TEST_SOURCES "${PROJECT_SOURCE_DIR}/test/*.cpp") +file(GLOB_RECURSE MILVUS_TEST_SOURCES "${PROJECT_SOURCE_DIR}/test/*.cpp") add_executable( milvus_test - ${BUSTUB_TEST_SOURCES} + ${MILVUS_TEST_SOURCES} ) target_link_libraries( - milvus_test milvus-storage GTest::gtest_main + milvus_test milvus-storage GTest::gtest_main GTest::gmock_main ) include(GoogleTest) diff --git a/cpp/test/manifest_test.cpp b/cpp/test/manifest_test.cpp index e4b315b..2d0f6c5 100644 --- a/cpp/test/manifest_test.cpp +++ b/cpp/test/manifest_test.cpp @@ -17,9 +17,9 @@ #include #include "gtest/gtest.h" #include "storage/manifest.h" -#include "gmock/gmock.h" #include "google/protobuf/util/message_differencer.h" #include +#include "gmock/gmock.h" using ::testing::ElementsAre; diff --git a/cpp/thirdparty/CMakeLists.txt b/cpp/thirdparty/CMakeLists.txt new file mode 100644 index 0000000..8f1456f --- /dev/null +++ b/cpp/thirdparty/CMakeLists.txt @@ -0,0 +1,3 @@ +add_subdirectory(gtest) +add_subdirectory(boost) +add_subdirectory(protobuf) \ No newline at end of file diff --git a/cpp/thirdparty/boost/CMakeLists.txt b/cpp/thirdparty/boost/CMakeLists.txt new file mode 100644 index 0000000..b85cefd --- /dev/null +++ b/cpp/thirdparty/boost/CMakeLists.txt @@ -0,0 +1,12 @@ +include(FetchContent) + +message(STATUS "Fetching Boost 1.84.0") +FetchContent_Declare( + Boost + URL https://github.com/boostorg/boost/releases/download/boost-1.84.0/boost-1.84.0.tar.xz + URL_MD5 893b5203b862eb9bbd08553e24ff146a + DOWNLOAD_EXTRACT_TIMESTAMP ON +) + +set(BOOST_INCLUDE_LIBRARIES uuid algorithm filesystem) +FetchContent_MakeAvailable(Boost) diff --git a/cpp/thirdparty/gtest/CMakeLists.txt b/cpp/thirdparty/gtest/CMakeLists.txt new file mode 100644 index 0000000..72c16a8 --- /dev/null +++ b/cpp/thirdparty/gtest/CMakeLists.txt @@ -0,0 +1,10 @@ +include(FetchContent) + +message(STATUS "Fetching googletest") +FetchContent_Declare( + googletest + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG v1.13.0 +) + +FetchContent_MakeAvailable(googletest) diff --git a/cpp/thirdparty/protobuf/CMakeLists.txt b/cpp/thirdparty/protobuf/CMakeLists.txt new file mode 100644 index 0000000..f76e01a --- /dev/null +++ b/cpp/thirdparty/protobuf/CMakeLists.txt @@ -0,0 +1,11 @@ +include(FetchContent) + +message(STATUS "Fetching protobuf") +FetchContent_Declare(protobuf + GIT_REPOSITORY https://github.com/protocolbuffers/protobuf.git + GIT_TAG v3.21.4 +) + +set(protobuf_BUILD_TESTS OFF CACHE INTERNAL "") +set(protobuf_BUILD_PROTOC_BINARIES ON CACHE INTERNAL "") +FetchContent_MakeAvailable(protobuf) \ No newline at end of file