build.yml

name: Build and Test
on:
  push:
    branches: ["**"]
    tags: [v**]
    paths:
      - .github/workflows/*.yml
      - build_tooling/**
      - cpp/**
      - python/**
      - setup.*
      - pyproject.toml
  workflow_dispatch:
    inputs:
      persistent_storage:
        description: Run the persistent storage tests
        type: boolean
      pypi_publish:
        type: boolean
      publish_env:
        description: Environment to publish to
        type: environment
      cmake_preset_type:
        description: Override CMAKE preset type
        type: choice
        options: ["-", debug, release]
run-name: Building ${{github.ref_name}} on ${{github.event_name}} by ${{github.actor}}
concurrency:
  group: ${{github.ref}}
  cancel-in-progress: true
jobs:
  cibw_docker_image:
    uses: ./.github/workflows/cibw_docker_image.yml
    permissions: {packages: write}
    with:
      cibuildwheel_ver: "2.12.1"
      force_update: false

  common_config:
    needs: [cibw_docker_image]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        include:
          # Use a matrix to build the common hierarchical structure used by multiple matrix jobs in build_steps.yaml
          # Please declare any key you added below in build_steps.yaml's dummy matrix as well to aid the linting tools
          - linux_matrix:
              - os: linux
                distro: ubuntu-latest
                cmake_preset_prefix: linux
                cibw_build_suffix: manylinux_x86_64
                build_dir: /tmp/cpp_build
                vcpkg_packages_dir: ""
                symbols: "*.gz" # Please update publish.yml when changing this!!
                do_not_archive: ["*.so", "*.[ao]", "vcpkg_installed"]
                test_services:
                  mongodb:
                    image: "mongo:4.4"
                  mongodb_2:
                    image: "mongo:4.4"
                container:
                  image: ${{needs.cibw_docker_image.outputs.tag}}
                  volumes:
                    - /:/mnt
            windows_matrix:
              - os: windows
                distro: windows-latest
                cmake_preset_prefix: windows-cl
                cibw_build_suffix: win_amd64
                build_dir: C:/cpp_build
                vcpkg_packages_dir: C:/vcpkg_packages
                symbols: "*.pdb" # Please update publish.yml when changing this!!
                do_not_archive: ["*.lib", "*.ilk", "*.pyd", "*.dll", "CMakeFiles/*.dir", "vcpkg_installed"]
                test_services: {}
                container: null
    steps:
      - run: |
          if ${{startsWith(github.ref, 'refs/tags/v')}}; then
            # Can only upload to Pypi once per version, so only auto upload on tag builds:
            echo -e "PUBLISH_ENV=ProdPypi\nCMAKE_PRESET_TYPE=release\nPYPI_PUBLISH=1" | tee -a $GITHUB_ENV
          elif $GITHUB_REF_PROTECTED || ${{github.ref == 'refs/heads/master'}} ; then
            echo -e "PUBLISH_ENV=TestPypi\nCMAKE_PRESET_TYPE=release" | tee -a $GITHUB_ENV
          else
            echo -e "PUBLISH_ENV=${{vars.DEFAULT_PUBLISH_ENV}}\nCMAKE_PRESET_TYPE=$DEFAULT_PRESET" | tee -a $GITHUB_ENV
          fi
        env:
          DEFAULT_PRESET: ${{startsWith(github.repository, 'man-group/ArcticDB') && 'release' || vars.DEFAULT_CMAKE_PRESET_TYPE || 'debug'}}
    outputs:
      cibuildwheel_ver: ${{needs.cibw_docker_image.outputs.cibuildwheel_ver}}
      publish_env: ${{inputs.publish_env || env.PUBLISH_ENV || ''}}
      pypi_publish: ${{inputs.pypi_publish || env.PYPI_PUBLISH}}
      cmake_preset_type_resolved: ${{inputs.cmake_preset_type != '-' && inputs.cmake_preset_type || env.CMAKE_PRESET_TYPE}}
      linux_matrix: ${{toJson(matrix.linux_matrix)}}
      windows_matrix: ${{toJson(matrix.windows_matrix)}}

  pre_seed_cleanup:
    if: inputs.persistent_storage == true
    name: Cleanup persistent storages
    uses: ./.github/workflows/persistent_storage.yml
    secrets: inherit
    with:
      job_type: cleanup

  persistent_storage_seed_linux:
    needs: [common_config, pre_seed_cleanup]
    if: inputs.persistent_storage == true
    strategy:
      fail-fast: false
      matrix:
        python3: ${{fromJson(vars.LINUX_PYTHON_VERSIONS || '[6, 7, 8, 9, 10, 11]')}}
        arcticdb_version: ["oldest", "latest"]
        include:
          - python_deps_ids: [""]
            matrix_override: ${{fromJson(needs.common_config.outputs.linux_matrix)}}
    name: Seed Persistent Storage for 3.${{matrix.python3}} Linux with ${{ matrix.arcticdb_version }} ArcticDB package version
    uses: ./.github/workflows/persistent_storage.yml
    secrets: inherit
    permissions: {packages: write}
    with:
      job_type: seed
      python3: ${{matrix.python3}}
      arcticdb_version: ${{matrix.arcticdb_version}}
      matrix: ${{toJson(matrix.matrix_override)}}
      python_deps_ids: ${{toJson(matrix.python_deps_ids)}}

  persistent_storage_seed_windows:
    needs: [common_config, pre_seed_cleanup]
    if: inputs.persistent_storage == true
    strategy:
      fail-fast: false
      matrix:
        python3: ${{fromJson(vars.WINDOWS_PYTHON_VERSIONS || '[7, 8, 9, 10, 11]')}}
        arcticdb_version: ["oldest", "latest"]
        include:
          - python_deps_ids: [""]
            matrix_override: ${{fromJson(needs.common_config.outputs.windows_matrix)}}
    name: Seed Persistent Storage for 3.${{matrix.python3}} Windows with ${{ matrix.arcticdb_version }} ArcticDB package version 
    uses: ./.github/workflows/persistent_storage.yml
    secrets: inherit
    permissions: {packages: write}
    with:
      job_type: seed
      python3: ${{matrix.python3}}
      arcticdb_version: ${{matrix.arcticdb_version}}
      matrix: ${{toJson(matrix.matrix_override)}}
      python_deps_ids: ${{toJson(matrix.python_deps_ids)}}

  leader-compile-linux:
    # First do the C++ core compilation using one Python version to seed the compilation caches (and fail quicker)
    needs: [cibw_docker_image, common_config]
    name: Linux Compile
    uses: ./.github/workflows/build_steps.yml
    secrets: inherit
    permissions: {packages: write}
    with:
      job_type: leader-compile
      cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}}
      cibw_image_tag: ${{needs.cibw_docker_image.outputs.tag}}
      matrix: ${{needs.common_config.outputs.linux_matrix}}

  leader-cpp-test-linux:
    # Compile and run the C++ tests separately concurrently with the following job
    needs: [leader-compile-linux, cibw_docker_image, common_config]
    name: Linux C++ Tests
    uses: ./.github/workflows/build_steps.yml
    secrets: inherit
    with:
      job_type: cpp-tests
      cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}}
      cibw_image_tag: ${{needs.cibw_docker_image.outputs.tag}}
      matrix: ${{needs.common_config.outputs.linux_matrix}}

  follower-linux:
    # Then use the cached compilation artifacts to build other python versions concurrently in cibuildwheels
    needs: [leader-compile-linux, cibw_docker_image, common_config]
    strategy:
      fail-fast: false
      matrix:
        python3: ${{fromJson(vars.LINUX_PYTHON_VERSIONS || '[6, 7, 8, 9, 10, 11]')}}
        include:
          - python_deps_ids: [""]
            matrix_override: ${{fromJson(needs.common_config.outputs.linux_matrix)}}
          - python3: 6
            python_deps_ids: ["", -compat36]
            matrix_override:
              - ${{fromJson(needs.common_config.outputs.linux_matrix)[0]}}
              - python_deps_id: -compat36
                python_deps: requirements-compatibility-py36.txt
          - python3: 8
            python_deps_ids: ["", -compat38]
            matrix_override:
              - ${{fromJson(needs.common_config.outputs.linux_matrix)[0]}}
              - python_deps_id: -compat38
                python_deps: requirements-compatibility-py38.txt
    name: 3.${{matrix.python3}} Linux
    uses: ./.github/workflows/build_steps.yml
    secrets: inherit
    permissions: {packages: write}
    with:
      job_type: follower
      python3: ${{matrix.python3}}
      cibw_image_tag: ${{needs.cibw_docker_image.outputs.tag}}
      cibw_version: ${{needs.common_config.outputs.cibuildwheel_ver}}
      cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}}
      matrix: ${{toJson(matrix.matrix_override)}}
      python_deps_ids: ${{toJson(matrix.python_deps_ids)}}
      persistent_storage: ${{inputs.persistent_storage}}
  
  leader-compile-windows:
    needs: [common_config]
    name: Windows Compile
    uses: ./.github/workflows/build_steps.yml
    secrets: inherit
    permissions: {packages: write}
    with:
      job_type: leader-compile
      cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}}
      matrix: ${{needs.common_config.outputs.windows_matrix}}

  leader-cpp-test-windows:
    needs: [leader-compile-windows, common_config]
    name: Windows C++ Tests
    uses: ./.github/workflows/build_steps.yml
    secrets: inherit
    with:
      job_type: cpp-tests
      cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}}
      matrix: ${{needs.common_config.outputs.windows_matrix}}

  follower-windows:
    needs: [leader-compile-windows, common_config]
    strategy:
      fail-fast: false
      matrix:
        python3: ${{fromJson(vars.WINDOWS_PYTHON_VERSIONS || '[7, 8, 9, 10, 11]')}}
    name: 3.${{matrix.python3}} Windows
    uses: ./.github/workflows/build_steps.yml
    secrets: inherit
    permissions: {packages: write}
    with:
      job_type: follower
      python3: ${{matrix.python3}}
      cibw_version: ${{needs.common_config.outputs.cibuildwheel_ver}}
      cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}}
      matrix: ${{needs.common_config.outputs.windows_matrix}}
      persistent_storage: ${{ inputs.persistent_storage }}

  persistent_storage_verify_linux:
    needs: [common_config, follower-linux, follower-windows]
    if: inputs.persistent_storage == true
    strategy:
      fail-fast: false
      matrix:
        python3: ${{fromJson(vars.LINUX_PYTHON_VERSIONS || '[6, 7, 8, 9, 10, 11]')}}
        arcticdb_version: ["oldest", "latest"]
        include:
          - python_deps_ids: [""]
            matrix_override: ${{fromJson(needs.common_config.outputs.linux_matrix)}}
    name: Verify Persistent Storage for 3.${{matrix.python3}} Linux with ${{ matrix.arcticdb_version }} ArcticDB package version 
    uses: ./.github/workflows/persistent_storage.yml
    secrets: inherit
    permissions: {packages: write}
    with:
      job_type: verify
      python3: ${{matrix.python3}}
      arcticdb_version: ${{matrix.arcticdb_version}}
      matrix: ${{toJson(matrix.matrix_override)}}
      python_deps_ids: ${{toJson(matrix.python_deps_ids)}}

  persistent_storage_verify_windows:
    needs: [common_config, follower-windows, follower-linux]
    if: inputs.persistent_storage == true
    strategy:
      fail-fast: false
      matrix:
        python3: ${{fromJson(vars.WINDOWS_PYTHON_VERSIONS || '[7, 8, 9, 10, 11]')}}
        arcticdb_version: ["oldest", "latest"]
        include:
          - python_deps_ids: [""]
            matrix_override: ${{fromJson(needs.common_config.outputs.windows_matrix)}}
    name: Verify Persistent Storage for 3.${{matrix.python3}} Windows with ${{ matrix.arcticdb_version }} ArcticDB package version 
    uses: ./.github/workflows/persistent_storage.yml
    secrets: inherit
    permissions: {packages: write}
    with:
      job_type: verify
      python3: ${{matrix.python3}}
      arcticdb_version: ${{matrix.arcticdb_version}}
      matrix: ${{toJson(matrix.matrix_override)}}
      python_deps_ids: ${{toJson(matrix.python_deps_ids)}}
    
  post_verify_cleanup:
    needs: [persistent_storage_verify_windows, persistent_storage_verify_linux]
    if: inputs.persistent_storage == true
    name: Cleanup persistent storages
    uses: ./.github/workflows/persistent_storage.yml
    secrets: inherit
    with:
      job_type: cleanup

  can_merge:
    needs: [leader-cpp-test-linux, leader-cpp-test-windows, follower-linux, follower-windows, persistent_storage_verify_linux, persistent_storage_verify_windows]
    if: |
      always() &&
      !failure() &&
      !cancelled()
    runs-on: ubuntu-latest
    steps:
      - run: echo Dummy job to simplify PR merge checks configuration
      # FUTURE: add some test stats/reporting

  publish:
    needs: [common_config, can_merge]
    if: |
      always() &&
      !failure() &&
      !cancelled() &&
      (needs.common_config.outputs.pypi_publish && needs.common_config.outputs.publish_env)
    uses: ./.github/workflows/publish.yml
    secrets: inherit
    permissions: {contents: write}
    with:
      environment: ${{needs.common_config.outputs.publish_env}}

  docs:
    if: ${{ false }}
    needs: [common_config, follower-linux] # Needs a Linux wheel to build the API docs
    uses: ./.github/workflows/docs.yml
    secrets: inherit
    with:
      environment: ${{needs.common_config.outputs.publish_env}}