Building lmdb-fixes on push by poodlewars #3882
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Test | |
| on: | |
| push: | |
| branches: ["**"] | |
| tags: [v**] | |
| paths: | |
| - .github/workflows/*.yml | |
| - build_tooling/** | |
| - cpp/** | |
| - python/** | |
| - setup.* | |
| - pyproject.toml | |
| workflow_dispatch: | |
| inputs: | |
| persistent_storage: | |
| description: Run the persistent storage tests | |
| type: boolean | |
| pypi_publish: | |
| type: boolean | |
| publish_env: | |
| description: Environment to publish to | |
| type: environment | |
| cmake_preset_type: | |
| description: Override CMAKE preset type | |
| type: choice | |
| options: ["-", debug, release] | |
| run-name: Building ${{github.ref_name}} on ${{github.event_name}} by ${{github.actor}} | |
| concurrency: | |
| group: ${{github.ref}} | |
| cancel-in-progress: true | |
| jobs: | |
| cibw_docker_image: | |
| uses: ./.github/workflows/cibw_docker_image.yml | |
| permissions: {packages: write} | |
| with: | |
| cibuildwheel_ver: "2.12.1" | |
| force_update: false | |
| common_config: | |
| needs: [cibw_docker_image] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| include: | |
| # Use a matrix to build the common hierarchical structure used by multiple matrix jobs in build_steps.yaml | |
| # Please declare any key you added below in build_steps.yaml's dummy matrix as well to aid the linting tools | |
| - linux_matrix: | |
| - os: linux | |
| distro: ubuntu-latest | |
| cmake_preset_prefix: linux | |
| cibw_build_suffix: manylinux_x86_64 | |
| build_dir: /tmp/cpp_build | |
| vcpkg_packages_dir: "" | |
| symbols: "*.gz" # Please update publish.yml when changing this!! | |
| do_not_archive: ["*.so", "*.[ao]", "vcpkg_installed"] | |
| test_services: | |
| mongodb: | |
| image: "mongo:4.4" | |
| mongodb_2: | |
| image: "mongo:4.4" | |
| container: | |
| image: ${{needs.cibw_docker_image.outputs.tag}} | |
| volumes: | |
| - /:/mnt | |
| windows_matrix: | |
| - os: windows | |
| distro: windows-latest | |
| cmake_preset_prefix: windows-cl | |
| cibw_build_suffix: win_amd64 | |
| build_dir: C:/cpp_build | |
| vcpkg_packages_dir: C:/vcpkg_packages | |
| symbols: "*.pdb" # Please update publish.yml when changing this!! | |
| do_not_archive: ["*.lib", "*.ilk", "*.pyd", "*.dll", "CMakeFiles/*.dir", "vcpkg_installed"] | |
| test_services: {} | |
| container: null | |
| steps: | |
| - run: | | |
| if ${{startsWith(github.ref, 'refs/tags/v')}}; then | |
| # Can only upload to Pypi once per version, so only auto upload on tag builds: | |
| echo -e "PUBLISH_ENV=ProdPypi\nCMAKE_PRESET_TYPE=release\nPYPI_PUBLISH=1" | tee -a $GITHUB_ENV | |
| elif $GITHUB_REF_PROTECTED || ${{github.ref == 'refs/heads/master'}} ; then | |
| echo -e "PUBLISH_ENV=TestPypi\nCMAKE_PRESET_TYPE=release" | tee -a $GITHUB_ENV | |
| else | |
| echo -e "PUBLISH_ENV=${{vars.DEFAULT_PUBLISH_ENV}}\nCMAKE_PRESET_TYPE=$DEFAULT_PRESET" | tee -a $GITHUB_ENV | |
| fi | |
| env: | |
| DEFAULT_PRESET: ${{startsWith(github.repository, 'man-group/ArcticDB') && 'release' || vars.DEFAULT_CMAKE_PRESET_TYPE || 'debug'}} | |
| outputs: | |
| cibuildwheel_ver: ${{needs.cibw_docker_image.outputs.cibuildwheel_ver}} | |
| publish_env: ${{inputs.publish_env || env.PUBLISH_ENV || ''}} | |
| pypi_publish: ${{inputs.pypi_publish || env.PYPI_PUBLISH}} | |
| cmake_preset_type_resolved: ${{inputs.cmake_preset_type != '-' && inputs.cmake_preset_type || env.CMAKE_PRESET_TYPE}} | |
| linux_matrix: ${{toJson(matrix.linux_matrix)}} | |
| windows_matrix: ${{toJson(matrix.windows_matrix)}} | |
| pre_seed_cleanup: | |
| if: inputs.persistent_storage == true | |
| name: Cleanup persistent storages | |
| uses: ./.github/workflows/persistent_storage.yml | |
| secrets: inherit | |
| with: | |
| job_type: cleanup | |
| persistent_storage_seed_linux: | |
| needs: [common_config, pre_seed_cleanup] | |
| if: inputs.persistent_storage == true | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python3: ${{fromJson(vars.LINUX_PYTHON_VERSIONS || '[6, 7, 8, 9, 10, 11]')}} | |
| arcticdb_version: ["oldest", "latest"] | |
| include: | |
| - python_deps_ids: [""] | |
| matrix_override: ${{fromJson(needs.common_config.outputs.linux_matrix)}} | |
| name: Seed Persistent Storage for 3.${{matrix.python3}} Linux with ${{ matrix.arcticdb_version }} ArcticDB package version | |
| uses: ./.github/workflows/persistent_storage.yml | |
| secrets: inherit | |
| permissions: {packages: write} | |
| with: | |
| job_type: seed | |
| python3: ${{matrix.python3}} | |
| arcticdb_version: ${{matrix.arcticdb_version}} | |
| matrix: ${{toJson(matrix.matrix_override)}} | |
| python_deps_ids: ${{toJson(matrix.python_deps_ids)}} | |
| persistent_storage_seed_windows: | |
| needs: [common_config, pre_seed_cleanup] | |
| if: inputs.persistent_storage == true | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python3: ${{fromJson(vars.WINDOWS_PYTHON_VERSIONS || '[7, 8, 9, 10, 11]')}} | |
| arcticdb_version: ["oldest", "latest"] | |
| include: | |
| - python_deps_ids: [""] | |
| matrix_override: ${{fromJson(needs.common_config.outputs.windows_matrix)}} | |
| name: Seed Persistent Storage for 3.${{matrix.python3}} Windows with ${{ matrix.arcticdb_version }} ArcticDB package version | |
| uses: ./.github/workflows/persistent_storage.yml | |
| secrets: inherit | |
| permissions: {packages: write} | |
| with: | |
| job_type: seed | |
| python3: ${{matrix.python3}} | |
| arcticdb_version: ${{matrix.arcticdb_version}} | |
| matrix: ${{toJson(matrix.matrix_override)}} | |
| python_deps_ids: ${{toJson(matrix.python_deps_ids)}} | |
| leader-compile-linux: | |
| # First do the C++ core compilation using one Python version to seed the compilation caches (and fail quicker) | |
| needs: [cibw_docker_image, common_config] | |
| name: Linux Compile | |
| uses: ./.github/workflows/build_steps.yml | |
| secrets: inherit | |
| permissions: {packages: write} | |
| with: | |
| job_type: leader-compile | |
| cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}} | |
| cibw_image_tag: ${{needs.cibw_docker_image.outputs.tag}} | |
| matrix: ${{needs.common_config.outputs.linux_matrix}} | |
| leader-cpp-test-linux: | |
| # Compile and run the C++ tests separately concurrently with the following job | |
| needs: [leader-compile-linux, cibw_docker_image, common_config] | |
| name: Linux C++ Tests | |
| uses: ./.github/workflows/build_steps.yml | |
| secrets: inherit | |
| with: | |
| job_type: cpp-tests | |
| cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}} | |
| cibw_image_tag: ${{needs.cibw_docker_image.outputs.tag}} | |
| matrix: ${{needs.common_config.outputs.linux_matrix}} | |
| follower-linux: | |
| # Then use the cached compilation artifacts to build other python versions concurrently in cibuildwheels | |
| needs: [leader-compile-linux, cibw_docker_image, common_config] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python3: ${{fromJson(vars.LINUX_PYTHON_VERSIONS || '[6, 7, 8, 9, 10, 11]')}} | |
| include: | |
| - python_deps_ids: [""] | |
| matrix_override: ${{fromJson(needs.common_config.outputs.linux_matrix)}} | |
| - python3: 6 | |
| python_deps_ids: ["", -compat36] | |
| matrix_override: | |
| - ${{fromJson(needs.common_config.outputs.linux_matrix)[0]}} | |
| - python_deps_id: -compat36 | |
| python_deps: requirements-compatibility-py36.txt | |
| - python3: 8 | |
| python_deps_ids: ["", -compat38] | |
| matrix_override: | |
| - ${{fromJson(needs.common_config.outputs.linux_matrix)[0]}} | |
| - python_deps_id: -compat38 | |
| python_deps: requirements-compatibility-py38.txt | |
| name: 3.${{matrix.python3}} Linux | |
| uses: ./.github/workflows/build_steps.yml | |
| secrets: inherit | |
| permissions: {packages: write} | |
| with: | |
| job_type: follower | |
| python3: ${{matrix.python3}} | |
| cibw_image_tag: ${{needs.cibw_docker_image.outputs.tag}} | |
| cibw_version: ${{needs.common_config.outputs.cibuildwheel_ver}} | |
| cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}} | |
| matrix: ${{toJson(matrix.matrix_override)}} | |
| python_deps_ids: ${{toJson(matrix.python_deps_ids)}} | |
| persistent_storage: ${{inputs.persistent_storage}} | |
| leader-compile-windows: | |
| needs: [common_config] | |
| name: Windows Compile | |
| uses: ./.github/workflows/build_steps.yml | |
| secrets: inherit | |
| permissions: {packages: write} | |
| with: | |
| job_type: leader-compile | |
| cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}} | |
| matrix: ${{needs.common_config.outputs.windows_matrix}} | |
| leader-cpp-test-windows: | |
| needs: [leader-compile-windows, common_config] | |
| name: Windows C++ Tests | |
| uses: ./.github/workflows/build_steps.yml | |
| secrets: inherit | |
| with: | |
| job_type: cpp-tests | |
| cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}} | |
| matrix: ${{needs.common_config.outputs.windows_matrix}} | |
| follower-windows: | |
| needs: [leader-compile-windows, common_config] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python3: ${{fromJson(vars.WINDOWS_PYTHON_VERSIONS || '[7, 8, 9, 10, 11]')}} | |
| name: 3.${{matrix.python3}} Windows | |
| uses: ./.github/workflows/build_steps.yml | |
| secrets: inherit | |
| permissions: {packages: write} | |
| with: | |
| job_type: follower | |
| python3: ${{matrix.python3}} | |
| cibw_version: ${{needs.common_config.outputs.cibuildwheel_ver}} | |
| cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}} | |
| matrix: ${{needs.common_config.outputs.windows_matrix}} | |
| persistent_storage: ${{ inputs.persistent_storage }} | |
| persistent_storage_verify_linux: | |
| needs: [common_config, follower-linux, follower-windows] | |
| if: inputs.persistent_storage == true | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python3: ${{fromJson(vars.LINUX_PYTHON_VERSIONS || '[6, 7, 8, 9, 10, 11]')}} | |
| arcticdb_version: ["oldest", "latest"] | |
| include: | |
| - python_deps_ids: [""] | |
| matrix_override: ${{fromJson(needs.common_config.outputs.linux_matrix)}} | |
| name: Verify Persistent Storage for 3.${{matrix.python3}} Linux with ${{ matrix.arcticdb_version }} ArcticDB package version | |
| uses: ./.github/workflows/persistent_storage.yml | |
| secrets: inherit | |
| permissions: {packages: write} | |
| with: | |
| job_type: verify | |
| python3: ${{matrix.python3}} | |
| arcticdb_version: ${{matrix.arcticdb_version}} | |
| matrix: ${{toJson(matrix.matrix_override)}} | |
| python_deps_ids: ${{toJson(matrix.python_deps_ids)}} | |
| persistent_storage_verify_windows: | |
| needs: [common_config, follower-windows, follower-linux] | |
| if: inputs.persistent_storage == true | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python3: ${{fromJson(vars.WINDOWS_PYTHON_VERSIONS || '[7, 8, 9, 10, 11]')}} | |
| arcticdb_version: ["oldest", "latest"] | |
| include: | |
| - python_deps_ids: [""] | |
| matrix_override: ${{fromJson(needs.common_config.outputs.windows_matrix)}} | |
| name: Verify Persistent Storage for 3.${{matrix.python3}} Windows with ${{ matrix.arcticdb_version }} ArcticDB package version | |
| uses: ./.github/workflows/persistent_storage.yml | |
| secrets: inherit | |
| permissions: {packages: write} | |
| with: | |
| job_type: verify | |
| python3: ${{matrix.python3}} | |
| arcticdb_version: ${{matrix.arcticdb_version}} | |
| matrix: ${{toJson(matrix.matrix_override)}} | |
| python_deps_ids: ${{toJson(matrix.python_deps_ids)}} | |
| post_verify_cleanup: | |
| needs: [persistent_storage_verify_windows, persistent_storage_verify_linux] | |
| if: inputs.persistent_storage == true | |
| name: Cleanup persistent storages | |
| uses: ./.github/workflows/persistent_storage.yml | |
| secrets: inherit | |
| with: | |
| job_type: cleanup | |
| can_merge: | |
| needs: [leader-cpp-test-linux, leader-cpp-test-windows, follower-linux, follower-windows, persistent_storage_verify_linux, persistent_storage_verify_windows] | |
| if: | | |
| always() && | |
| !failure() && | |
| !cancelled() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - run: echo Dummy job to simplify PR merge checks configuration | |
| # FUTURE: add some test stats/reporting | |
| publish: | |
| needs: [common_config, can_merge] | |
| if: | | |
| always() && | |
| !failure() && | |
| !cancelled() && | |
| (needs.common_config.outputs.pypi_publish && needs.common_config.outputs.publish_env) | |
| uses: ./.github/workflows/publish.yml | |
| secrets: inherit | |
| permissions: {contents: write} | |
| with: | |
| environment: ${{needs.common_config.outputs.publish_env}} | |
| docs: | |
| if: ${{ false }} | |
| needs: [common_config, follower-linux] # Needs a Linux wheel to build the API docs | |
| uses: ./.github/workflows/docs.yml | |
| secrets: inherit | |
| with: | |
| environment: ${{needs.common_config.outputs.publish_env}} |