Integration tests #10667
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration tests | |
| concurrency: | |
| group: '${{ github.workflow }}-${{ github.ref }}' | |
| cancel-in-progress: '${{ github.ref != ''refs/heads/main'' }}' | |
| 'on': | |
| workflow_dispatch: null | |
| merge_group: null | |
| pull_request: | |
| branches: | |
| - main | |
| push: | |
| branches: | |
| - main | |
| schedule: | |
| - cron: 0 0 * * * | |
| env: | |
| CI: 1 | |
| CARGO_INCREMENTAL: 0 | |
| CACHE_TIMEOUT_MINUTES: 5 | |
| SCRIPT_TIMEOUT_MINUTES: 30 | |
| AWS_ACCESS_KEY_ID: '${{ secrets.AWS_ACCESS_KEY_ID }}' | |
| AWS_SECRET_ACCESS_KEY: '${{ secrets.AWS_SECRET_ACCESS_KEY }}' | |
| RUSTC_WRAPPER: sccache | |
| CC: sccache clang | |
| CXX: sccache clang++ | |
| FIL_PROOFS_PARAMETER_CACHE: /var/tmp/filecoin-proof-parameters | |
| SHELL_IMAGE: busybox | |
| jobs: | |
| build-macos: | |
| name: Build MacOS | |
| runs-on: macos-latest | |
| steps: | |
| - name: Configure SCCache variables | |
| run: | | |
| # External PRs do not have access to 'vars' or 'secrets'. | |
| if [[ "$AWS_ACCESS_KEY_ID" != "" ]]; then | |
| echo "SCCACHE_ENDPOINT=${{ vars.SCCACHE_ENDPOINT}}" >> $GITHUB_ENV | |
| echo "SCCACHE_BUCKET=${{ vars.SCCACHE_BUCKET}}" >> $GITHUB_ENV | |
| echo "SCCACHE_REGION=${{ vars.SCCACHE_REGION}}" >> $GITHUB_ENV | |
| fi | |
| - name: Checkout Sources | |
| uses: actions/checkout@v4 | |
| - name: Setup sccache | |
| uses: mozilla-actions/[email protected] | |
| timeout-minutes: '${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}' | |
| continue-on-error: true | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: "go.work" | |
| - name: Cargo Check | |
| run: cargo check | |
| build-ubuntu: | |
| name: Build Ubuntu | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Configure SCCache variables | |
| run: | | |
| # External PRs do not have access to 'vars' or 'secrets'. | |
| if [[ "${{secrets.AWS_ACCESS_KEY_ID}}" != "" ]]; then | |
| echo "SCCACHE_ENDPOINT=${{ vars.SCCACHE_ENDPOINT}}" >> $GITHUB_ENV | |
| echo "SCCACHE_BUCKET=${{ vars.SCCACHE_BUCKET}}" >> $GITHUB_ENV | |
| echo "SCCACHE_REGION=${{ vars.SCCACHE_REGION}}" >> $GITHUB_ENV | |
| fi | |
| - run: lscpu | |
| - name: Show IP | |
| run: curl ifconfig.me | |
| continue-on-error: true | |
| - name: Checkout Sources | |
| uses: actions/checkout@v4 | |
| - name: Setup sccache | |
| uses: mozilla-actions/[email protected] | |
| timeout-minutes: '${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}' | |
| continue-on-error: true | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: "go.work" | |
| - name: Cargo Install | |
| env: | |
| # To minimize compile times: https://nnethercote.github.io/perf-book/build-configuration.html#minimizing-compile-times | |
| RUSTFLAGS: "-C linker=clang -C link-arg=-fuse-ld=lld" | |
| run: make install-slim-quick | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: 'forest-${{ runner.os }}' | |
| path: | | |
| ~/.cargo/bin/forest* | |
| if-no-files-found: error | |
| cargo-publish-dry-run: | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Checkout Sources | |
| uses: actions/checkout@v4 | |
| - name: Setup sccache | |
| uses: mozilla-actions/[email protected] | |
| timeout-minutes: '${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}' | |
| continue-on-error: true | |
| - name: Install Apt Dependencies | |
| run: | | |
| sudo make install-deps | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: "go.work" | |
| - run: cargo publish --dry-run | |
| forest-cli-check: | |
| needs: | |
| - build-ubuntu | |
| name: Forest CLI checks | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - run: lscpu | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.FIL_PROOFS_PARAMETER_CACHE }} | |
| key: proof-params-keys | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: 'forest-${{ runner.os }}' | |
| path: ~/.cargo/bin | |
| - name: Set permissions | |
| run: | | |
| chmod +x ~/.cargo/bin/forest* | |
| - name: forest-cli check | |
| run: ./scripts/tests/forest_cli_check.sh | |
| timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}' | |
| calibnet-check: | |
| needs: | |
| - build-ubuntu | |
| name: Calibnet check | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - run: lscpu | |
| - uses: actions/cache@v4 | |
| with: | |
| path: '${{ env.FIL_PROOFS_PARAMETER_CACHE }}' | |
| key: proof-params-keys | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: 'forest-${{ runner.os }}' | |
| path: ~/.cargo/bin | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: 'forest-${{ runner.os }}' | |
| path: ~/.cargo/bin | |
| - name: Set permissions | |
| run: | | |
| chmod +x ~/.cargo/bin/forest* | |
| - name: Other commands check | |
| run: ./scripts/tests/calibnet_other_check.sh | |
| timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}' | |
| env: | |
| CC: clang | |
| CXX: clang++ | |
| calibnet-stateless-mode-check: | |
| needs: | |
| - build-ubuntu | |
| name: Calibnet stateless mode check | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - run: lscpu | |
| - uses: actions/cache@v4 | |
| with: | |
| path: '${{ env.FIL_PROOFS_PARAMETER_CACHE }}' | |
| key: proof-params-keys | |
| - name: Checkout Sources | |
| uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: 'forest-${{ runner.os }}' | |
| path: ~/.cargo/bin | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: 'forest-${{ runner.os }}' | |
| path: ~/.cargo/bin | |
| - name: Set permissions | |
| run: | | |
| chmod +x ~/.cargo/bin/forest* | |
| - run: ./scripts/tests/calibnet_stateless_mode_check.sh | |
| timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}' | |
| state-migrations-check: | |
| needs: | |
| - build-ubuntu | |
| name: State migrations | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - run: lscpu | |
| - uses: actions/cache@v4 | |
| with: | |
| path: '${{ env.FIL_PROOFS_PARAMETER_CACHE }}' | |
| key: proof-params-keys | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: 'forest-${{ runner.os }}' | |
| path: ~/.cargo/bin | |
| - name: Set permissions | |
| run: | | |
| chmod +x ~/.cargo/bin/forest* | |
| - name: Migration Regression Tests | |
| run: ./scripts/tests/calibnet_migration_regression_tests.sh | |
| timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}' | |
| calibnet-wallet-check: | |
| needs: | |
| - build-ubuntu | |
| name: Wallet tests | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - run: lscpu | |
| - uses: actions/cache@v4 | |
| with: | |
| path: '${{ env.FIL_PROOFS_PARAMETER_CACHE }}' | |
| key: proof-params-keys | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: 'forest-${{ runner.os }}' | |
| path: ~/.cargo/bin | |
| - name: Set permissions | |
| run: | | |
| chmod +x ~/.cargo/bin/forest* | |
| - name: Wallet commands check | |
| env: | |
| CALIBNET_WALLET: '${{ secrets.CALIBNET_WALLET }}' | |
| run: | | |
| if [[ "$CALIBNET_WALLET" != "" ]]; then | |
| ./scripts/tests/calibnet_wallet_check.sh "$CALIBNET_WALLET" | |
| fi | |
| timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}' | |
| calibnet-export-check: | |
| needs: | |
| - build-ubuntu | |
| name: Snapshot export checks | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - run: lscpu | |
| - uses: actions/cache@v4 | |
| with: | |
| path: '${{ env.FIL_PROOFS_PARAMETER_CACHE }}' | |
| key: proof-params-keys | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: 'forest-${{ runner.os }}' | |
| path: ~/.cargo/bin | |
| - name: Set permissions | |
| run: | | |
| chmod +x ~/.cargo/bin/forest* | |
| - name: Snapshot export check | |
| run: ./scripts/tests/calibnet_export_check.sh | |
| timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}' | |
| calibnet-no-discovery-checks: | |
| needs: | |
| - build-ubuntu | |
| name: Calibnet no discovery checks | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - run: lscpu | |
| - uses: actions/cache@v4 | |
| with: | |
| path: '${{ env.FIL_PROOFS_PARAMETER_CACHE }}' | |
| key: proof-params-keys | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: 'forest-${{ runner.os }}' | |
| path: ~/.cargo/bin | |
| - name: Set permissions | |
| run: | | |
| chmod +x ~/.cargo/bin/forest* | |
| - run: ./scripts/tests/calibnet_no_discovery_check.sh | |
| timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}' | |
| calibnet-kademlia-checks: | |
| needs: | |
| - build-ubuntu | |
| name: Calibnet kademlia checks | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - run: lscpu | |
| - uses: actions/cache@v4 | |
| with: | |
| path: '${{ env.FIL_PROOFS_PARAMETER_CACHE }}' | |
| key: proof-params-keys | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: 'forest-${{ runner.os }}' | |
| path: ~/.cargo/bin | |
| - name: Set permissions | |
| run: | | |
| chmod +x ~/.cargo/bin/forest* | |
| - run: ./scripts/tests/calibnet_kademlia_check.sh | |
| timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}' | |
| calibnet-eth-mapping-check: | |
| needs: | |
| - build-ubuntu | |
| name: Calibnet eth mapping check | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - run: lscpu | |
| - uses: actions/cache@v4 | |
| with: | |
| path: '${{ env.FIL_PROOFS_PARAMETER_CACHE }}' | |
| key: proof-params-keys | |
| - name: Checkout Sources | |
| uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: 'forest-${{ runner.os }}' | |
| path: ~/.cargo/bin | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: 'forest-${{ runner.os }}' | |
| path: ~/.cargo/bin | |
| - name: Set permissions | |
| run: | | |
| chmod +x ~/.cargo/bin/forest* | |
| - run: ./scripts/tests/calibnet_eth_mapping_check.sh | |
| timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}' | |
| db-migration-checks: | |
| needs: | |
| - build-ubuntu | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/cache@v4 | |
| with: | |
| path: '${{ env.FIL_PROOFS_PARAMETER_CACHE }}' | |
| key: proof-params-keys | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: 'forest-${{ runner.os }}' | |
| path: ~/.cargo/bin | |
| - name: Set permissions | |
| run: | | |
| chmod +x ~/.cargo/bin/forest* | |
| - name: Database migration checks | |
| run: ./scripts/tests/calibnet_db_migration.sh | |
| timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}' | |
| db-migration-checks-car-db: | |
| needs: | |
| - build-ubuntu | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/cache@v4 | |
| with: | |
| path: '${{ env.FIL_PROOFS_PARAMETER_CACHE }}' | |
| key: proof-params-keys | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: 'forest-${{ runner.os }}' | |
| path: ~/.cargo/bin | |
| - name: Set permissions | |
| run: | | |
| chmod +x ~/.cargo/bin/forest* | |
| - name: Database migration checks with car_db folder | |
| run: ./scripts/tests/calibnet_db_migration_car_db.sh | |
| timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}' | |
| local-devnet-check: | |
| name: Devnet checks | |
| runs-on: ubuntu-24.04 | |
| needs: | |
| - build-ubuntu | |
| env: | |
| # We use a custom Dockerfile for CI to speed up the build process. | |
| FOREST_DOCKERFILE_OVERRIDE: scripts/devnet/forest_ci.dockerfile | |
| steps: | |
| - uses: actions/cache@v4 | |
| id: cache | |
| with: | |
| path: '${{ env.FIL_PROOFS_PARAMETER_CACHE }}' | |
| key: proof-params-2k | |
| - name: Load cache into volume | |
| if: steps.cache.outputs.cache-hit == 'true' | |
| run: | | |
| docker volume create devnet_filecoin-proofs | |
| docker run --rm \ | |
| -v devnet_filecoin-proofs:/proofs \ | |
| -v $FIL_PROOFS_PARAMETER_CACHE:/cache \ | |
| $SHELL_IMAGE \ | |
| sh -c "cp /cache/* /proofs" | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: 'forest-${{ runner.os }}' | |
| - name: Devnet setup | |
| run: ./scripts/devnet/setup.sh | |
| timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}' | |
| - name: Devnet check | |
| run: ./scripts/devnet/check.sh | |
| timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}' | |
| - name: Dump docker logs | |
| if: always() | |
| uses: jwalton/gh-docker-logs@v2 | |
| - name: Prepare cache folder for uploading | |
| if: steps.cache.outputs.cache-hit != 'true' | |
| run: | | |
| docker run --rm \ | |
| -v devnet_filecoin-proofs:/proofs \ | |
| -v $FIL_PROOFS_PARAMETER_CACHE:/cache \ | |
| $SHELL_IMAGE \ | |
| sh -c "cp /proofs/* /cache" | |
| sudo chmod -R 755 $FIL_PROOFS_PARAMETER_CACHE | |
| calibnet-rpc-checks: | |
| needs: | |
| - build-ubuntu | |
| name: Calibnet RPC checks | |
| runs-on: ubuntu-24.04 | |
| env: | |
| # We use a custom Dockerfile for CI to speed up the build process. | |
| FOREST_DOCKERFILE_OVERRIDE: scripts/devnet/forest_ci.dockerfile | |
| steps: | |
| # Remove some unnecessary software to free up space. This should free up around 15-20 GB. | |
| # This is required because of the limited space on the runner, | |
| # and disk space-hungry snapshots used in the setup. | |
| # This is taken from: | |
| # https://github.com/easimon/maximize-build-space/blob/fc881a613ad2a34aca9c9624518214ebc21dfc0c/action.yml#L121-L136 | |
| # Using the action directly is not feasible as it does some more modifications that break the setup in our case. | |
| - name: Remove unnecessary software | |
| run: | | |
| echo "Disk space before cleanup" | |
| df -h | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /usr/local/lib/android | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf /opt/hostedtoolcache/CodeQL | |
| echo "Disk space after cleanup" | |
| df -h | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: 'forest-${{ runner.os }}' | |
| - name: Run api compare tests | |
| run: ./scripts/tests/api_compare/setup.sh | |
| timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}' | |
| - name: Dump docker logs | |
| if: always() | |
| uses: jwalton/gh-docker-logs@v2 | |
| - name: Dump Docker usage | |
| if: always() | |
| run: | | |
| docker system df | |
| docker system df --verbose | |
| df -h | |
| bootstrap-checks-forest: | |
| needs: | |
| - build-ubuntu | |
| name: Bootstrap checks - Forest | |
| runs-on: ubuntu-24.04 | |
| env: | |
| # We use a custom Dockerfile for CI to speed up the build process. | |
| FOREST_DOCKERFILE_OVERRIDE: scripts/devnet/forest_ci.dockerfile | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: 'forest-${{ runner.os }}' | |
| - name: Run bootstrap tests | |
| run: ./scripts/tests/bootstrapper/test_bootstrapper.sh forest | |
| timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}' | |
| - name: Dump docker logs | |
| if: always() | |
| uses: jwalton/gh-docker-logs@v2 | |
| bootstrap-checks-lotus: | |
| needs: | |
| - build-ubuntu | |
| name: Bootstrap checks - Lotus | |
| runs-on: ubuntu-24.04 | |
| env: | |
| # We use a custom Dockerfile for CI to speed up the build process. | |
| FOREST_DOCKERFILE_OVERRIDE: scripts/devnet/forest_ci.dockerfile | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: 'forest-${{ runner.os }}' | |
| - name: Run bootstrap tests | |
| run: ./scripts/tests/bootstrapper/test_bootstrapper.sh lotus | |
| timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}' | |
| - name: Dump docker logs | |
| if: always() | |
| uses: jwalton/gh-docker-logs@v2 | |
| snapshot-parity-checks: | |
| needs: | |
| - build-ubuntu | |
| name: Calibnet snapshot parity checks | |
| runs-on: buildjet-8vcpu-ubuntu-2204 | |
| # Run the job only on main to limit the runner cost. | |
| # The downside is that the offending commit can only be caught after being merged | |
| # and a manual revert is required. | |
| # Note that running this step on non-draft PR does not save the cost as it always re-run | |
| # all the jobs when a PR is moved out of draft. Also this parity test will be stale soon | |
| # once we start including the F3(fast finality) metadata in the Forest snapshots. | |
| if: github.ref == 'refs/heads/main' | |
| env: | |
| # We use a custom Dockerfile for CI to speed up the build process. | |
| FOREST_DOCKERFILE_OVERRIDE: scripts/devnet/forest_ci.dockerfile | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: 'forest-${{ runner.os }}' | |
| - name: Run snapshot parity tests | |
| run: ./scripts/tests/snapshot_parity/setup.sh | |
| timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}' | |
| - name: Dump docker logs | |
| if: always() | |
| uses: jwalton/gh-docker-logs@v2 |