conda-build: Add 'windows-latest' to conda build workflow

.github/workflows/build_with_conda.yml

@@ -36,10 +36,10 @@ jobs:
         uses: SimenB/[email protected]
         id: cpu-cores
 
-      - name: Install Conda environment from environment_unix.yml
+      - name: Install Conda environment from environment-dev.yml
         uses: mamba-org/[email protected]
         with:
-          environment-file: environment_unix.yml
+          environment-file: environment-dev.yml
           init-shell: >-
             bash
           cache-environment: true
@@ -111,10 +111,10 @@ jobs:
         uses: SimenB/[email protected]
         id: cpu-cores
 
-      - name: Install Conda environment from environment_unix.yml
+      - name: Install Conda environment from environment-dev.yml
         uses: mamba-org/[email protected]
         with:
-          environment-file: environment_unix.yml
+          environment-file: environment-dev.yml
           environment-name: arcticdb
           init-shell: >-
             bash
@@ -158,3 +158,49 @@ jobs:
         env:
           ARCTICDB_USING_CONDA: 1
 
+  win:
+    strategy:
+      matrix:
+        include:
+         - os: windows-latest
+
+      fail-fast: false
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Install Conda environment from environment-dev.yml
+        uses: mamba-org/provision-with-micromamba@main
+        with:
+          environment-file: environment-dev.yml
+          environment-name: arcticdb
+
+      - name: micromamba shell hook
+        shell: powershell
+        run: |
+          micromamba shell hook -s cmd.exe -p C:\Users\runneradmin\micromamba-root
+
+      - name: Make build directory
+        run: mkdir build
+
+      - name: cmake configure
+        shell: cmd
+        run: |
+          call C:\Users\runneradmin\micromamba-root\condabin\micromamba.bat activate arcticdb
+          # Protocol buffers compilation require not using build isolation.
+          python -m pip install --no-build-isolation --no-deps -v -e .
+        env:
+          ARCTICDB_USING_CONDA: 1
+
+      - name: Test with pytest
+        shell: cmd
+        run: |
+          call C:\Users\runneradmin\micromamba-root\condabin\micromamba.bat activate arcticdb
+          cd python
+          export ARCTICDB_RAND_SEED=$RANDOM
+          python -m pytest -n ${{ steps.cpu-cores.outputs.count }} tests
+        env:
+          ARCTICDB_USING_CONDA: 1

cpp/CMakeLists.txt

@@ -16,6 +16,8 @@ option(
 )
 
 
+set(CMAKE_VERBOSE_MAKEFILE ON CACHE BOOL "ON" FORCE)
+
 if(WIN32)
     add_compile_definitions(NOGDI)
     add_compile_definitions(WIN32_LEAN_AND_MEAN)
@@ -25,9 +27,23 @@ if(WIN32)
     # We always generate the debug info on Windows since those are not in the final .dll.
     # sccache needs the /Z7 format, but the linker will still produce .pdb files.
     # Guide to MSVC compilation warnings https://learn.microsoft.com/en-us/cpp/error-messages/compiler-warnings/compiler-warnings-c4000-through-c4199?view=msvc-170
-    set(CMAKE_CXX_FLAGS "/DWIN32 /D_WINDOWS /GR /EHsc /bigobj /Z7 /w /wd4244 /wd4267")
-    set(CMAKE_CXX_FLAGS_RELEASE "/MT /DNDEBUG /O2")
-    set(CMAKE_CXX_FLAGS_DEBUG "/MTd /RTC1 /Od")
+    if(${ARCTICDB_USING_CONDA})
+        set(CMAKE_CXX_FLAGS "/DWIN32 /D_WINDOWS /GR /EHsc /bigobj /Z7 /w /wd4244 /wd4267 /MP")
+        set(CMAKE_CXX_FLAGS_RELEASE "/MD /DNDEBUG")
+        set(CMAKE_CXX_FLAGS_DEBUG "/MD")
+
+        # `RelWithDebInfo` is used for conda-forge builds.
+        # The default for `CMAKE_CXX_FLAGS_RELWITHDEBINFO` is `/MD /Zi /O2 /Ob1 /DNDEBUG`.
+        # We don't want /Zi, because it uses .pdb files instead
+        # which are not distributed by some libraries like folly.
+        set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "/MD /Ob1 /DNDEBUG")
+    else()
+        set(CMAKE_CXX_FLAGS "/DWIN32 /D_WINDOWS /GR /EHsc /bigobj /Z7 /w /wd4244 /wd4267")
+        set(CMAKE_CXX_FLAGS_RELEASE "/MT /DNDEBUG /O2")
+        set(CMAKE_CXX_FLAGS_DEBUG "/MTd /RTC1 /Od")
+        set(CMAKE_CXX_FLAGS_RELEASE "/MT /DNDEBUG")
+        set(CMAKE_CXX_FLAGS_DEBUG "/MTd /RTC1")
+    endif()
 else()
     if(${ARCTICDB_USING_CONDA})
         set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra")

cpp/CMakePresets.json

@@ -60,7 +60,7 @@
     {
       "name": "windows",
       "hidden": true,
-      "generator": "Ninja",
+      "generator": "Visual Studio 17 2022",
       "environment": {"cmakepreset_expected_host_system": "Windows"}
     },
     {
@@ -83,7 +83,11 @@
     },
     {
       "name": "windows-cl-conda-debug",
-      "inherits": ["common_conda", "windows"]
+      "inherits": ["common_conda", "windows"],
+      "cacheVariables": {
+        "CMAKE_C_COMPILER": "cl",
+        "CMAKE_CXX_COMPILER": "cl"
+      }
     },
 
 
@@ -123,7 +127,7 @@
 
 
     { "name": "windows-cl-release",       "inherits": "windows-cl-debug",       "cacheVariables": { "CMAKE_BUILD_TYPE": "Release" }},
-    { "name": "windows-cl-conda-release", "inherits": "windows-cl-conda-debug", "cacheVariables": { "CMAKE_BUILD_TYPE": "Release" } },
+    { "name": "windows-cl-conda-release", "inherits": "windows-cl-conda-debug", "cacheVariables": { "CMAKE_BUILD_TYPE": "RelWithDebInfo" } },
     { "name": "linux-release",            "inherits": "linux-debug",            "cacheVariables": { "CMAKE_BUILD_TYPE": "RelWithDebInfo" } },
     { "name": "linux-conda-release",      "inherits": "linux-conda-debug",      "cacheVariables": { "CMAKE_BUILD_TYPE": "RelWithDebInfo" } },
     { "name": "darwin-conda-release",     "inherits": "darwin-conda-debug",     "cacheVariables": { "CMAKE_BUILD_TYPE": "RelWithDebInfo" } }

cpp/arcticdb/CMakeLists.txt

@@ -107,7 +107,10 @@ endif()
 
 # Libraries baked into the internal and external Linux images are found by searching various paths
 # On Windows, vcpkg will provide a "Config" which takes precedence
-find_library(Pcre_LIBRARY NAMES pcre PATHS /usr/local/lib/libpcre.a REQUIRED)
+if(NOT ${ARCTICDB_USING_CONDA} OR NOT WIN32)
+    find_library(Pcre_LIBRARY NAMES pcre PATHS /usr/local/lib/libpcre.a REQUIRED)
+endif()
+
 if (WIN32)
     find_package(Iconv)
 else ()
@@ -498,7 +501,7 @@ set (arcticdb_core_libraries
         prometheus-cpp::pull
         ${standard_libraries}
         fmt::fmt
-        ${Protobuf_lite_LIBRARY}
+	# ${Protobuf_lite_LIBRARY}
         ${Protobuf_LIBRARY}
         ${LZ4_LIBRARY}
         ${Zstd_LIBRARY}
@@ -516,7 +519,7 @@ if(${ARCTICDB_USING_CONDA})
     # We must use dynamic libraries.
     list (APPEND arcticdb_core_libraries
         xxHash::xxHash
-        pcre
+        ${PCRE_LIBRARIES}
         msgpack-c
         ${LMDB_LIBRARIES}
     )

cpp/vcpkg.json

@@ -145,7 +145,7 @@
     { "name": "curl", "version": "8.4.0" },
     { "name": "double-conversion", "version": "3.2.1" },
     { "name": "fmt", "version": "9.1.0#1" },
-    { "name": "folly", "version-string": "2022.10.31.00#3" },
+    { "name": "folly", "version-string": "2023.05.22.00" },
     { "name": "gflags", "version": "2.2.2#5" },
     { "name": "glog", "version": "0.6.0#2" },
     { "name": "gtest", "version": "1.12.1" },

environment_unix.yml → environment-dev.yml

@@ -9,11 +9,11 @@ dependencies:
   - gtest
   - gflags
   - doxygen
-  - boost-cpp
   - grpcio
   - grpcio-tools
   - protobuf
-  # Build dependencies
+  # Host dependencies
+  - boost-cpp
   - krb5
   - lz4-c
   - double-conversion
@@ -28,7 +28,7 @@ dependencies:
   - cyrus-sasl
   - aws-sdk-cpp
   - prometheus-cpp
-  - libprotobuf < 4
+  - libprotobuf
   - bitmagic
   - spdlog
   - azure-core-cpp
@@ -55,7 +55,7 @@ dependencies:
   # Build dependencies for tests
   - libarrow
   # Python dependences
-  - python =3.10 # Python 3.11 requires protobuf>=4
+  - python =3.10
   - packaging
   - numpy
   - pandas

setup.py

@@ -63,7 +63,8 @@ def run(self):
         print(f"\nProtoc compilation (into '{output_dir}') for versions '{self.proto_vers}':")
         for proto_ver in self.proto_vers:
             if python >= (3, 11) and proto_ver == "3":
-                print(f"Python protobuf {proto_ver} is not officially supported on Python {python}. Skipping...") # e.g. https://pypi.org/project/protobuf/3.20.3/#files
+                # e.g. https://pypi.org/project/protobuf/3.20.3/#files
+                print(f"Python protobuf {proto_ver} is not officially supported on Python {python}. Skipping...")
             elif python <= (3, 6) and proto_ver >= "4":
                 print(f"Python protobuf {proto_ver} do not run on Python {python}. Skipping...")
             else:
@@ -74,32 +75,17 @@ def _compile_one_version(self, proto_ver: str, version_output_dir: str):
         assert grpc_version, "Supported proto-vers arguments are " + ", ".join(self._PROTOBUF_TO_GRPC_VERSION)
 
         # Manual virtualenv to avoid hard-coding Man internal locations
-        pythonpath = mkdtemp()
-        if not ARCTICDB_USING_CONDA:
-            # Python protobuf 3 and 4 are incompatible and we do not want to dictate which version of protobuf
-            # the user can have, so we compile the Python binding files with both versions and dynamically load
-            # the correct version at run time.
-            _log_and_run(
-                sys.executable,
-                "-mpip",
-                "install",
-                "--disable-pip-version-check",
-                "--target=" + pythonpath,
-                "grpcio-tools" + grpc_version,
-                f"protobuf=={proto_ver}.*",
-            )
-            env = {**os.environ, "PYTHONPATH": pythonpath, "PYTHONNOUSERSITE": "1"}
-        else:
-            # grpcio-tools is already installed in the conda environment (see environment.yml)
-            env = {**os.environ}
+        # grpcio-tools and protobuf already are installed in the conda environment (see environment.yml)
+        # Using protoc installed in the conda environment from conda-forge.
+        env = {**os.environ}
+        cmd = ["protoc", "--version"]
+        _log_and_run(*cmd, env=env)
 
         # Compile
+        cmd = ["protoc", "-Icpp/proto", "--python_out=" + version_output_dir]
         os.makedirs(version_output_dir, exist_ok=True)
-        cmd = [sys.executable, "-mgrpc_tools.protoc", "-Icpp/proto", "--python_out=" + version_output_dir]
         _log_and_run(*cmd, *glob.glob(os.path.normpath("cpp/proto/arcticc/pb2/*.proto")), env=env)
 
-        shutil.rmtree(pythonpath)
-
 
 class CompileProtoAndBuild(build_py):
     def run(self):