Compile using cmake

.github/workflows/cpp-ci.yml

@@ -18,19 +18,24 @@ jobs:
       - name: Install dependencies
         uses: aminya/setup-cpp@v1
         with: 
-          conan: 1.58.0
+          # conan: 1.58.0
           cmake: true
 
-      - name: setup conan
-        run: 
-          conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install libcurl4-openssl-dev
+
+      # - name: setup conan
+      #   run: 
+      #     conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local
 
-      - name: conan package cache
-        uses: actions/cache@v3
-        with:
-          path: ~/.conan
-          key: conan-${{ hashFiles('./cpp/conanfile.py') }}
-          restore-keys: conan-
+      # - name: conan package cache
+      #   uses: actions/cache@v3
+      #   with:
+      #     path: ~/.conan
+      #     key: conan-${{ hashFiles('./cpp/conanfile.py') }}
+      #     restore-keys: conan-
 
       - name: Build
         working-directory: ./cpp
@@ -52,7 +57,7 @@ jobs:
           make check-tidy
 
       - name : Test
-        working-directory: ./cpp/build/Release
+        working-directory: ./cpp/build/test
         run: 
           ctest .
 

cpp/CMakeLists.txt

@@ -7,28 +7,33 @@ option(WITH_ASAN "Build with address sanitizer." OFF)
 option(WITH_OPENDAL "Build with opendal." OFF)
 
 set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 if (WITH_OPENDAL)
   add_compile_definitions(MILVUS_OPENDAL)
-  list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
   include(libopendal)
 endif()
 
-find_package(Boost REQUIRED)
-find_package(Arrow REQUIRED)
-find_package(protobuf REQUIRED)
-find_package(glog REQUIRED)
+include(libarrow)
+include(libglog)
+add_subdirectory(thirdparty)
 
+find_package(Protobuf REQUIRED)
+include_directories(${protobuf_INCLUDE_DIR})
 file(GLOB_RECURSE SRC_FILES src/*.cpp src/*.cc)
 add_library(milvus-storage ${SRC_FILES})
-target_include_directories(milvus-storage PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/milvus-storage ${CMAKE_CURRENT_SOURCE_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR}/test/include)
+target_include_directories(milvus-storage PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/milvus-storage ${CMAKE_CURRENT_SOURCE_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR}/test/include )
 set(LINK_LIBS 
-    arrow::libarrow
-    arrow::libparquet
-    Boost::boost
-    protobuf::protobuf
-    glog::glog)
+    libarrow
+    libparquet
+    Boost::uuid
+    Boost::algorithm
+    protobuf
+    Boost::filesystem
+    libglog
+    )
 
 if (WITH_OPENDAL)
   list(APPEND LINK_LIBS opendal)
@@ -37,6 +42,5 @@ endif()
 target_link_libraries(milvus-storage PUBLIC ${LINK_LIBS})
 
 if (WITH_UT)
-  enable_testing()
   add_subdirectory(test)
 endif()

cpp/Makefile

@@ -7,19 +7,19 @@ endif
 
 build:
 	mkdir -p build && cd build && \
-	conan install .. --build=missing && \
-	conan build ..
+	cmake .. && \
+	make -j 8
 
 debug:
 	mkdir -p build && cd build && \
-	conan install .. --build=missing -s build_type=Debug && \
-	conan build ..
+	cmake -DCMAKE_BUILD_TYPE=Debug .. && \
+	make -j 8
 
 clean:
 	rm -rf build
 
 test: build
-	cd build/Release && ctest . -j ${TEST_THREADS} --output-on-failure
+	cd build/test && ctest . -j ${TEST_THREADS} --output-on-failure
 
 fix-format:
 	find ./src -type f ! -name "*.pb.h"  -iname *.h -o -iname *.cpp | xargs clang-format -i
@@ -32,10 +32,10 @@ check-format:
 	find ./test -type f ! -name "*.pb.h"  -iname *.h -o -iname *.cpp | xargs clang-format --dry-run --Werror
 
 check-tidy:
-	python3 ./scripts/run-clang-tidy.py -p build/Release
+	python3 ./scripts/run-clang-tidy.py -p build 
 
 fix-tidy: 
-	python3 ./scripts/run-clang-tidy.py -fix -p build/Release
+	python3 ./scripts/run-clang-tidy.py -fix -p build
 
 proto: 
 	protoc -I="src/proto" --cpp_out="src/proto" src/proto/*.proto

cpp/cmake/libarrow.cmake

@@ -0,0 +1,46 @@
+function(build_arrow)
+    include(ExternalProject)
+    set(ARROW_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/thirdparty/arrow-ep)
+
+    file(MAKE_DIRECTORY 
+        ${ARROW_PREFIX}
+        "${ARROW_PREFIX}/include"
+        "${ARROW_PREFIX}/lib"
+    )
+    ExternalProject_Add(
+        arrow_ep
+        GIT_REPOSITORY https://github.com/apache/arrow.git
+        GIT_TAG apache-arrow-16.0.0
+        CMAKE_ARGS 
+            -DARROW_PARQUET=ON
+            -DARROW_FILESYSTEM=ON
+            -DARROW_S3=ON
+            -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
+        SOURCE_SUBDIR cpp
+        INSTALL_DIR ${ARROW_PREFIX}
+        )
+
+    ExternalProject_Get_Property(arrow_ep install_dir)
+
+    message(STATUS ${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/release/libparquet.so)
+    add_library(libarrow SHARED IMPORTED)
+    set_target_properties(libarrow
+        PROPERTIES
+        IMPORTED_GLOBAL TRUE
+        IMPORTED_LOCATION   ${ARROW_PREFIX}/lib/libarrow.so
+        INTERFACE_INCLUDE_DIRECTORIES ${install_dir}/include/
+        )
+
+    add_library(libparquet SHARED IMPORTED)
+    set_target_properties(libparquet
+        PROPERTIES 
+        IMPORTED_GLOBAL TRUE
+        IMPORTED_LOCATION   ${ARROW_PREFIX}/lib/libparquet.so
+        INTERFACE_INCLUDE_DIRECTORIES ${install_dir}/include/
+        )
+
+    add_dependencies(libarrow arrow_ep)
+    add_dependencies(libparquet arrow_ep)
+endfunction()
+
+build_arrow()

cpp/cmake/libglog.cmake

@@ -0,0 +1,32 @@
+function(build_glog)
+    include(ExternalProject)
+    set(GLOG_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/thirdparty/glog-ep)
+
+    file(MAKE_DIRECTORY 
+        ${GLOG_PREFIX}
+        "${GLOG_PREFIX}/include"
+        "${GLOG_PREFIX}/lib"
+    )
+    ExternalProject_Add(
+        glog_ep
+        GIT_REPOSITORY https://github.com/google/glog.git
+        GIT_TAG v0.6.0
+        CMAKE_ARGS 
+            -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
+        INSTALL_DIR ${GLOG_PREFIX}
+        )
+
+    ExternalProject_Get_Property(glog_ep install_dir)
+
+    add_library(libglog SHARED IMPORTED)
+    set_target_properties(libglog
+        PROPERTIES
+        IMPORTED_GLOBAL TRUE
+        IMPORTED_LOCATION   ${GLOG_PREFIX}/lib/libglog.so
+        INTERFACE_INCLUDE_DIRECTORIES ${install_dir}/include/
+        )
+
+    add_dependencies(libglog glog_ep)
+endfunction()
+
+build_glog()

cpp/test/CMakeLists.txt

@@ -1,14 +1,14 @@
-find_package(GTest REQUIRED)
+enable_testing()
 
-file(GLOB_RECURSE BUSTUB_TEST_SOURCES "${PROJECT_SOURCE_DIR}/test/*.cpp")
+file(GLOB_RECURSE MILVUS_TEST_SOURCES "${PROJECT_SOURCE_DIR}/test/*.cpp")
 
 add_executable(
   milvus_test
-  ${BUSTUB_TEST_SOURCES}
+  ${MILVUS_TEST_SOURCES}
 )
 
 target_link_libraries(
-  milvus_test milvus-storage GTest::gtest_main
+  milvus_test milvus-storage GTest::gtest_main GTest::gmock_main 
 )
 
 include(GoogleTest)

cpp/test/manifest_test.cpp

@@ -17,9 +17,9 @@
 #include <numeric>
 #include "gtest/gtest.h"
 #include "storage/manifest.h"
-#include "gmock/gmock.h"
 #include "google/protobuf/util/message_differencer.h"
 #include <arrow/util/key_value_metadata.h>
+#include "gmock/gmock.h"
 
 using ::testing::ElementsAre;
 

cpp/thirdparty/CMakeLists.txt

@@ -0,0 +1,3 @@
+add_subdirectory(gtest)
+add_subdirectory(boost)
+add_subdirectory(protobuf)

cpp/thirdparty/boost/CMakeLists.txt

@@ -0,0 +1,12 @@
+include(FetchContent)
+
+message(STATUS "Fetching Boost 1.84.0")
+FetchContent_Declare(
+  Boost
+  URL https://github.com/boostorg/boost/releases/download/boost-1.84.0/boost-1.84.0.tar.xz
+  URL_MD5 893b5203b862eb9bbd08553e24ff146a
+  DOWNLOAD_EXTRACT_TIMESTAMP ON
+)
+
+set(BOOST_INCLUDE_LIBRARIES uuid algorithm filesystem)
+FetchContent_MakeAvailable(Boost)

cpp/thirdparty/gtest/CMakeLists.txt

@@ -0,0 +1,10 @@
+include(FetchContent)
+
+message(STATUS "Fetching googletest")
+FetchContent_Declare(
+  googletest
+  GIT_REPOSITORY https://github.com/google/googletest.git
+  GIT_TAG v1.13.0
+)
+
+FetchContent_MakeAvailable(googletest)

cpp/thirdparty/protobuf/CMakeLists.txt

@@ -0,0 +1,11 @@
+include(FetchContent)
+
+message(STATUS "Fetching protobuf")
+FetchContent_Declare(protobuf
+  GIT_REPOSITORY https://github.com/protocolbuffers/protobuf.git
+  GIT_TAG        v3.21.4
+)
+
+set(protobuf_BUILD_TESTS OFF CACHE INTERNAL "")
+set(protobuf_BUILD_PROTOC_BINARIES ON CACHE INTERNAL "")
+FetchContent_MakeAvailable(protobuf)