WIP: build multiplier only using vcpkg

.github/workflows/ci.yml

@@ -15,9 +15,9 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  LLVM_PASTA_VER: ba5b66d
   LLVM_VER: 17
   PYTHON_VER: 3.11
+  VCPKG_BINARY_SOURCES: clear;files,/root/vcpkg-binary-cache,readwrite
 
 jobs:
   build:
@@ -58,42 +58,38 @@ jobs:
       - name: Set up variables
         run: |
           echo "RELEASE_DIR=${RUNNER_TEMP}/release" >> $GITHUB_ENV
-          echo "LLVM_PREFIX_DIR=${RUNNER_TEMP}/llvm" >> $GITHUB_ENV
           echo "VERSION=$(git rev-parse --short=7 HEAD)" >> $GITHUB_ENV
           echo "ARTIFACT_NAME=multiplier-$(git rev-parse --short=7 HEAD).tar.xz" >> $GITHUB_ENV
+          echo "CC=$(which clang-$LLVM_VER)" >> $GITHUB_ENV
+          echo "CXX=$(which clang++-$LLVM_VER)" >> $GITHUB_ENV
+          echo "CFLAGS=-g0 --ld-path=$(which ld.lld-$LLVM_VER) -flto" >> $GITHUB_ENV
+          echo "CXXFLAGS=-g0 --ld-path=$(which ld.lld-$LLVM_VER) -flto" >> $GITHUB_ENV
 
-      - name: Install llvm-pasta
+      - name: Setup vcpkg
         run: |
-          mkdir -p ${RELEASE_DIR}
-          mkdir -p ${LLVM_PREFIX_DIR}
-          gh release download ${LLVM_PASTA_VER} --repo trail-of-forks/llvm-project --pattern 'llvm-*'
-          tar -xvJf llvm-pasta-*.tar.xz -C ${LLVM_PREFIX_DIR}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          mkdir -p /root/vcpkg-binary-cache
+          git clone https://github.com/microsoft/vcpkg $HOME/vcpkg
+          $HOME/vcpkg/bootstrap-vcpkg.sh
+
+      - name: Cache vcpkg binaries
+        id: cache-vcpkg
+        uses: actions/cache@v4
+        with:
+          path: /root/vcpkg-binary-cache
+          key: ${{ runner.os }}-vcpkg-binaries
 
       - name: Build
         run: |
-
-          CXXFLAGS="-g0" \
-          CCFLAGS="-g0" \
           cmake \
+            -DVCPKG_TARGET_TRIPLET=x64-linux-rel \
+            --toolchain "$HOME/vcpkg/scripts/buildsystems/vcpkg.cmake" \
             -DCMAKE_BUILD_TYPE=Release \
-            -DCMAKE_INSTALL_PREFIX=${RELEASE_DIR} \
-            -DCMAKE_EXE_LINKER_FLAGS_INIT="-fuse-ld=$(which ld.lld-${LLVM_VER})" \
-            -DCMAKE_MODULE_LINKER_FLAGS_INIT="-fuse-ld=$(which ld.lld-${LLVM_VER})" \
-            -DCMAKE_SHARED_LINKER_FLAGS_INIT="-fuse-ld=$(which ld.lld-${LLVM_VER})" \
-            -DCMAKE_C_COMPILER="$(which clang-${LLVM_VER})" \
-            -DCMAKE_CXX_COMPILER=$(which clang++-${LLVM_VER}) \
-            -DClang_DIR=${LLVM_PREFIX_DIR}/lib/cmake/clang \
-            -DLLVM_DIR=${LLVM_PREFIX_DIR}/lib/cmake/llvm \
-            -DMLIR_DIR=${LLVM_PREFIX_DIR}/lib/cmake/mlir \
-            -DMX_USE_VENDORED_CLANG=OFF \
-            -DMX_USE_VENDORED_PASTA=OFF \
-            -DLLVM_ENABLE_LLD:BOOL=TRUE \
             -GNinja \
-            ./
+            -S . \
+            -B build
 
-          ninja install
+          cmake --build build
+          cmake --install build --prefix ${RELEASE_DIR}
 
       # Release on merge to main
       - name: Package Release

.gitmodules

@@ -1,53 +0,0 @@
-[submodule "vendor/concurrentqueue/src"]
-	path = vendor/concurrentqueue/src
-	url = https://github.com/cameron314/concurrentqueue.git
-	shallow = true
-[submodule "vendor/reproc/src"]
-	path = vendor/reproc/src
-	url = https://github.com/DaanDeMeyer/reproc.git
-	shallow = true
-[submodule "vendor/capnproto/src"]
-	path = vendor/capnproto/src
-	url = https://github.com/capnproto/capnproto.git
-	shallow = true
-[submodule "vendor/glog/src"]
-	path = vendor/glog/src
-	url = https://github.com/google/glog.git
-	shallow = true
-[submodule "vendor/gflags/src"]
-	path = vendor/gflags/src
-	url = https://github.com/gflags/gflags.git
-	shallow = true
-[submodule "vendor/zstd/src"]
-	path = vendor/zstd/src
-	url = https://github.com/facebook/zstd.git
-	shallow = true
-[submodule "vendor/gap/src"]
-	path = vendor/gap/src
-	url = https://github.com/lifting-bits/gap.git
-	shallow = true
-[submodule "vendor/pasta/src"]
-	path = vendor/pasta/src
-	url = https://github.com/trailofbits/pasta.git
-	branch = master
-[submodule "vendor/llvm-project/src"]
-	path = vendor/llvm-project/src
-	url = https://github.com/trail-of-forks/llvm-project.git
-	branch = pasta-llvmorg-17.0.4
-[submodule "vendor/abseil/src"]
-	path = vendor/abseil/src
-	url = https://github.com/abseil/abseil-cpp.git
-	shallow = true
-[submodule "vendor/re2/src"]
-	path = vendor/re2/src
-	url = https://github.com/google/re2.git
-	shallow = true
-[submodule "vendor/rocksdb/src"]
-	path = vendor/rocksdb/src
-	url = https://github.com/facebook/rocksdb.git
-[submodule "vendor/xxhash/src"]
-	path = vendor/xxhash/src
-	url = https://github.com/Cyan4973/xxHash.git
-[submodule "vendor/vast/src"]
-	path = vendor/vast/src
-	url = https://github.com/trailofbits/vast.git

CMakeLists.txt

@@ -25,7 +25,7 @@ endif()
 list(PREPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/modules")
 
 include("cmake/system.cmake")
-include("cmake/llvm.cmake")
+#include("cmake/llvm.cmake")
 include("cmake/linker.cmake")
 include("cmake/git.cmake")
 

bin/Index/CMakeLists.txt

@@ -73,6 +73,8 @@ add_executable("${exe_name}"
   "${MX_BOOTSTRAP_INCLUDE_VISITOR_INC_H}"
 )
 
+find_package(concurrentqueue MODULE REQUIRED)
+
 target_link_libraries("${exe_name}"
   PRIVATE
     "mx-api"
@@ -118,7 +120,7 @@ set_target_properties("${exe_name}"
       "${PROJECT_BINARY_DIR}/${MX_INSTALL_BIN_DIR}"
 )
 
-find_and_link_llvm_dependencies("${exe_name}")
+#find_and_link_llvm_dependencies("${exe_name}")
 
 if(MX_ENABLE_INSTALL AND NOT MX_ENABLE_BOOTSTRAP)
   install(

bindings/Python/CMakeLists.txt

@@ -117,7 +117,7 @@ if(MX_ENABLE_INSTALL AND NOT MX_ENABLE_BOOTSTRAP)
     DIRECTORY
       "${CMAKE_CURRENT_LIST_DIR}/${lower_project_name}-stubs"
     DESTINATION
-      "${CMAKE_INSTALL_PREFIX}/${MX_INSTALL_LIB_DIR}/${site_packages_dir}"
+      "${MX_INSTALL_LIB_DIR}/${site_packages_dir}"
   )
 
   # Install the binary into the target site-packages directory.

cmake/modules/Findconcurrentqueue.cmake

@@ -0,0 +1,16 @@
+find_path(concurrentqueue_INCLUDE_DIR concurrentqueue.h PATH_SUFFIXES concurrentqueue)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(concurrentqueue
+    REQUIRED_VARS concurrentqueue_INCLUDE_DIR)
+
+# Create the imported target
+if(concurrentqueue_FOUND)
+    set(concurrentqueue_INCLUDE_DIRS ${concurrentqueue_INCLUDE_DIR})
+
+    if(NOT TARGET concurrentqueue)
+        add_library(concurrentqueue INTERFACE IMPORTED)
+        set_target_properties(concurrentqueue PROPERTIES
+            INTERFACE_INCLUDE_DIRECTORIES "${concurrentqueue_INCLUDE_DIRS}")
+    endif()
+endif()

cmake/options.cmake

@@ -16,21 +16,6 @@ set(MX_INSTALL_INCLUDE_DIR "${CMAKE_INSTALL_INCLUDEDIR}" CACHE PATH "Directory i
 set(MX_INSTALL_SHARE_DIR "${CMAKE_INSTALL_DATADIR}" CACHE PATH "Directory in which Multiplier CMake files will be installed")
 set(MX_VENDOR_INSTALL_DIR "${PROJECT_BINARY_DIR}/vendor/install" CACHE PATH "Directory in which multiplier's vendored dependencies are installed")
 
-option(MX_USE_VENDORED_CAPNPROTO "Set to OFF to disable default building of Cap'n Proto as a vendored library." ON)
-option(MX_USE_VENDORED_GLOG "Set to OFF to disable default building of Google glog as a vendored library." ON)
-option(MX_USE_VENDORED_GFLAGS "Set to OFF to disable default building of gflags as a vendored library." ON)
-option(MX_USE_VENDORED_ZSTD "Set to OFF to disable default building of ZStandard as a vendored library." ON)
-option(MX_USE_VENDORED_REPROC "Set to OFF to disable default building of reproc/reproc++ as a vendored library." ON)
-option(MX_USE_VENDORED_GAP "Set to OFF to disable default building of gap as a vendored library." ON)
-option(MX_USE_VENDORED_CLANG "Set to OFF to disable default building of Clang/LLVM as a vendored library." ON)
-option(MX_USE_VENDORED_PASTA "Set to OFF to disable default building of PASTA as a vendored library." ON)
-option(MX_USE_VENDORED_ABSEIL "Set to OFF to disable default building of Abseil as a vendored library." ON)
-option(MX_USE_VENDORED_RE2 "Set to OFF to disable default building of RE2 as a vendored library." ON)
-option(MX_USE_VENDORED_ROCKSDB "Set to OFF to disable default building of RocksDB as a vendored library." ON)
-option(MX_USE_VENDORED_XXHASH "Set to OFF to disable default building of xxHash as a vendored library." ON)
-option(MX_USE_VENDORED_NANOBIND "Set to OFF to disable default building of nanobind as a vendored library." ON)
-option(MX_USE_VENDORED_VAST "Set to OFF to disable default building of VAST as a vendored library." ON)
-
 option(MX_ENABLE_BOOTSTRAP "Set to ON to enable re-bootstrapping of some auto-generated files." OFF)
 option(MX_ENABLE_INSTALL "Set to ON to enable the install target" ON)
 option(MX_ENABLE_RE2 "Set to OFF to disable RE2 integration" ON)

cmake/vast.cmake

@@ -11,4 +11,5 @@ set(VAST_LIBS
   VAST::VASTAliasTypeInterface
   VAST::VASTElementTypeInterface
   VAST::VASTCodeGen
+  VAST::VASTFrontend
 )

custom-ports/capnproto/portfile.cmake

@@ -0,0 +1,34 @@
+vcpkg_from_github(
+    OUT_SOURCE_PATH SOURCE_PATH
+    REPO capnproto/capnproto
+    REF f5531718b95c2ad4fcedc624e0d46866455007f4
+    SHA512 255c7a630096c322056775f98336df8d6ab0fb967226d9fca3db6c997dcb4e5580c9e3041aece7c7623ea006f2d857dea348399c9cc869ac8414cdfa36714a75
+    HEAD_REF master
+)
+
+vcpkg_cmake_configure(
+    SOURCE_PATH "${SOURCE_PATH}"
+    OPTIONS
+        -DCAPNP_LITE=OFF
+        -DEXTERNAL_CAPNP=OFF
+        -DBUILD_TESTING=OFF
+        -DWITH_ZLIB=AUTO
+        -DWITH_OPENSSL=AUTO
+        -DWITH_FIBERS=OFF
+)
+
+vcpkg_cmake_install()
+
+vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake/CapnProto)
+
+vcpkg_copy_tools(TOOL_NAMES capnp capnpc-c++ capnpc-capnp AUTO_CLEAN)
+
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin")
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/bin")
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include")
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share")
+
+# Handle copyright
+file(INSTALL "${SOURCE_PATH}/LICENSE" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright)
+
+vcpkg_fixup_pkgconfig()

custom-ports/capnproto/vcpkg.json

@@ -0,0 +1,9 @@
+{
+    "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json",
+    "name": "capnproto",
+    "version": "0.1.0",
+    "dependencies": [
+        "vcpkg-cmake",
+        "vcpkg-cmake-config"
+    ]
+}

custom-ports/gap/portfile.cmake

@@ -0,0 +1,27 @@
+vcpkg_from_github(
+    OUT_SOURCE_PATH SOURCE_PATH
+    REPO lifting-bits/gap
+    REF ad8fefaf7235a9cd6670e272ca4487807ed81f8a
+    SHA512 9e9259fd4c3c96e31965613092f7dec9df5e236aa4ef9ac122378fe1708d4efd2fb6bd9837530e8474646dc3b0aa409781fd6f45fc28929d07bb54a95a072566
+    HEAD_REF main
+)
+
+vcpkg_cmake_configure(
+    SOURCE_PATH "${SOURCE_PATH}"
+    OPTIONS
+        -DGAP_ENABLE_TESTING=OFF
+        -DGAP_ENABLE_EXAMPLES=OFF
+        -DGAP_ENABLE_COROUTINES=ON
+        -DGAP_ENABLE_WARNINGS=OFF
+        -DGAP_INSTALL=ON
+        -DUSE_SYSTEM_DEPENDENCIES=ON
+)
+
+vcpkg_cmake_install()
+vcpkg_cmake_config_fixup(
+    PACKAGE_NAME "gap"
+    CONFIG_PATH lib/cmake/gap
+)
+vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE")
+
+file( REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug" )

custom-ports/gap/vcpkg.json

@@ -0,0 +1,9 @@
+{
+    "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json",
+    "name": "gap",
+    "version": "0.1.0",
+    "dependencies": [
+        "vcpkg-cmake",
+        "vcpkg-cmake-config"
+    ]
+}