From 728b51e3e28dc5f3748e47cf0be61ca03dea7edd Mon Sep 17 00:00:00 2001 From: Ryan Beck-Buysse Date: Wed, 8 Jun 2022 13:58:16 -0500 Subject: [PATCH] Update GHA workflows to use ephemeral ec2 runners The hardware we were previously using for multi-arch builds is being end-of-lifed so this is the next best solution. Signed-off-by: Ryan Beck-Buysse --- .github/workflows/grid-dev.yaml | 80 +++++++- .github/workflows/publish-docker-branch.yaml | 195 ++++++++++++++++++- .github/workflows/publish-release.yaml | 86 +++++++- 3 files changed, 355 insertions(+), 6 deletions(-) diff --git a/.github/workflows/grid-dev.yaml b/.github/workflows/grid-dev.yaml index 4ec2f12b0f..f978f72d59 100644 --- a/.github/workflows/grid-dev.yaml +++ b/.github/workflows/grid-dev.yaml @@ -3,10 +3,54 @@ on: - push - workflow_dispatch jobs: + start_cluster: + if: github.repository == 'hyperledger/grid' + name: Start buildx cluster + runs-on: ubuntu-latest + outputs: + label: ${{ steps.start_buildx_cluster.outputs.label }} + permissions: + id-token: write + contents: read + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-region: ${{ secrets.AWS_BUILDX_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + + - uses: actions/checkout@v2 + + - name: Start EC2 runner + id: start_buildx_cluster + uses: ./.github/actions/ec2-runners + with: + action: start + amd_ami_id: ${{ secrets.AMD_AMI_ID }} + amd_instance_type: ${{ secrets.AMD_INSTANCE_TYPE }} + arm_ami_id: ${{ secrets.ARM_AMI_ID }} + arm_instance_type: ${{ secrets.ARM_INSTANCE_TYPE }} + gh_personal_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + security_group_id: ${{ secrets.SECURITY_GROUP_ID }} + subnet: ${{ secrets.SUBNET }} + + - name: Output label + run: echo ${{ steps.start_buildx_cluster.outputs.label }} + + - name: Notify Slack of Failure + if: cancelled() || failure() + uses: 8398a7/action-slack@v3 + with: + status: ${{ job.status }} + fields: repo,message,author,job + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + build_grid_dev: if: github.repository == 'hyperledger/grid' name: Build grid-dev - runs-on: macos-arm + needs: start_cluster + runs-on: ${{ needs.start_cluster.outputs.label }} steps: - name: Login to DockerHub uses: docker/login-action@v1 @@ -31,3 +75,37 @@ jobs: fields: repo,message,author,job env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + + stop_cluster: + name: Stop buildx cluster + needs: + - start_cluster + - build_grid_dev + runs-on: ubuntu-latest + if: ${{ github.repository == 'hyperledger/grid' && always() }} + permissions: + id-token: write + contents: read + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-region: ${{ secrets.AWS_BUILDX_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + + - uses: actions/checkout@v2 + + - name: Destroy cluster + uses: ./.github/actions/ec2-runners + with: + action: stop + label: ${{ needs.start_cluster.outputs.label }} + + - name: Notify Slack of Failure + if: cancelled() || failure() + uses: 8398a7/action-slack@v3 + with: + status: ${{ job.status }} + fields: repo,message,author,job + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/publish-docker-branch.yaml b/.github/workflows/publish-docker-branch.yaml index 6800b6fcc9..650874ffff 100644 --- a/.github/workflows/publish-docker-branch.yaml +++ b/.github/workflows/publish-docker-branch.yaml @@ -13,10 +13,56 @@ concurrency: cancel-in-progress: true jobs: - publish_docker_images: + start_cluster_nightly: if: >- github.repository_owner == 'hyperledger' - runs-on: macos-arm + name: Start buildx cluster + runs-on: ubuntu-latest + outputs: + label: ${{ steps.start-buildx-cluster.outputs.label }} + permissions: + id-token: write + contents: read + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-region: ${{ secrets.AWS_BUILDX_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + + - uses: actions/checkout@v2 + + - name: Start EC2 runner + id: start-buildx-cluster + uses: ./.github/actions/ec2-runners + with: + action: start + amd_ami_id: ${{ secrets.AMD_AMI_ID }} + amd_instance_type: c6i.4xlarge + arm_ami_id: ${{ secrets.ARM_AMI_ID }} + arm_instance_type: c6g.4xlarge + gh_personal_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + security_group_id: ${{ secrets.SECURITY_GROUP_ID }} + subnet: ${{ secrets.SUBNET }} + + - name: Output label + run: echo ${{ steps.start-buildx-cluster.outputs.label }} + + - name: Notify Slack of Failure + if: failure() + uses: 8398a7/action-slack@v3 + with: + status: ${{ job.status }} + fields: repo,message,author,job + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + + publish_docker_nightly: + if: >- + github.repository_owner == 'hyperledger' + needs: + - start_cluster_nightly + runs-on: ${{ needs.start_cluster_nightly.outputs.label }} steps: - name: Display envvars run: env @@ -31,13 +77,122 @@ jobs: with: fetch-depth: 0 - - name: Publish main images to dockerhub + - name: Publish nightly images to dockerhub run: docker buildx ls && ./ci/publish-docker nightly env: NAMESPACE: ${{ secrets.DOCKER_HUB_NAMESPACE }}/ VERSION: AUTO_STRICT CARGO_TERM_COLOR: always + - name: Notify Slack of Failure + if: failure() + uses: 8398a7/action-slack@v3 + with: + status: ${{ job.status }} + fields: repo,message,author,job + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + + stop_cluster_nightly: + if: ${{ github.repository == 'hyperledger/grid' && always() }} + name: Stop buildx cluster + needs: + - start_cluster_nightly + - publish_docker_nightly + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-region: ${{ secrets.AWS_BUILDX_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + + - uses: actions/checkout@v2 + + - name: Destroy cluster + uses: ./.github/actions/ec2-runners + with: + action: stop + label: ${{ needs.start_cluster_nightly.outputs.label }} + + - name: Notify Slack of Failure + if: failure() + uses: 8398a7/action-slack@v3 + with: + status: ${{ job.status }} + fields: repo,message,author,job + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + +# --== experimental images ==-- + + start_cluster_experimental: + if: >- + github.repository_owner == 'hyperledger' + name: Start buildx cluster + runs-on: ubuntu-latest + outputs: + label: ${{ steps.start-buildx-cluster.outputs.label }} + permissions: + id-token: write + contents: read + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-region: ${{ secrets.AWS_BUILDX_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + + - uses: actions/checkout@v2 + + - name: Start EC2 runner + id: start-buildx-cluster + uses: ./.github/actions/ec2-runners + with: + action: start + amd_ami_id: ${{ secrets.AMD_AMI_ID }} + amd_instance_type: c6i.4xlarge + arm_ami_id: ${{ secrets.ARM_AMI_ID }} + arm_instance_type: c6g.4xlarge + gh_personal_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + security_group_id: ${{ secrets.SECURITY_GROUP_ID }} + subnet: ${{ secrets.SUBNET }} + + - name: Output label + run: echo ${{ steps.start-buildx-cluster.outputs.label }} + + - name: Notify Slack of Failure + if: failure() + uses: 8398a7/action-slack@v3 + with: + status: ${{ job.status }} + fields: repo,message,author,job + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + + publish_docker_experimental: + if: >- + github.repository_owner == 'hyperledger' + needs: + - start_cluster_experimental + runs-on: ${{ needs.start_cluster_experimental.outputs.label }} + steps: + - name: Display envvars + run: env + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Publish experimental images to dockerhub run: docker buildx ls && ./ci/publish-docker experimental env: @@ -53,3 +208,37 @@ jobs: fields: repo,message,author,job env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + + stop_cluster_experimental: + if: ${{ github.repository == 'hyperledger/grid' && always() }} + name: Stop buildx cluster + needs: + - start_cluster_experimental + - publish_docker_experimental + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-region: ${{ secrets.AWS_BUILDX_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + + - uses: actions/checkout@v2 + + - name: Destroy cluster + uses: ./.github/actions/ec2-runners + with: + action: stop + label: ${{ needs.start_cluster_experimental.outputs.label }} + + - name: Notify Slack of Failure + if: failure() + uses: 8398a7/action-slack@v3 + with: + status: ${{ job.status }} + fields: repo,message,author,job + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/publish-release.yaml b/.github/workflows/publish-release.yaml index 9068f0bf52..40d5d135d4 100644 --- a/.github/workflows/publish-release.yaml +++ b/.github/workflows/publish-release.yaml @@ -28,11 +28,59 @@ jobs: - name: Run tests run: just ci-test + start_cluster: + needs: + - unit_test_grid + if: >- + github.repository_owner == 'hyperledger' + name: Start buildx cluster + runs-on: ubuntu-latest + outputs: + label: ${{ steps.start-buildx-cluster.outputs.label }} + permissions: + id-token: write + contents: read + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-region: ${{ secrets.AWS_BUILDX_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + + - uses: actions/checkout@v2 + + - name: Start EC2 runner + id: start-buildx-cluster + uses: ./.github/actions/ec2-runners + with: + action: start + amd_ami_id: ${{ secrets.AMD_AMI_ID }} + amd_instance_type: c6i.4xlarge + arm_ami_id: ${{ secrets.ARM_AMI_ID }} + arm_instance_type: c6g.4xlarge + gh_personal_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + security_group_id: ${{ secrets.SECURITY_GROUP_ID }} + subnet: ${{ secrets.SUBNET }} + + - name: Output label + run: echo ${{ steps.start-buildx-cluster.outputs.label }} + + - name: Notify Slack of Failure + if: failure() + uses: 8398a7/action-slack@v3 + with: + status: ${{ job.status }} + fields: repo,message,author,job + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + publish_docker: - needs: unit_test_grid + needs: + - start_cluster + - unit_test_grid if: >- github.repository_owner == 'hyperledger' - runs-on: macos-arm + runs-on: ${{ needs.start_cluster.outputs.label }} steps: - name: Display envvars run: env @@ -65,6 +113,40 @@ jobs: env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + stop_cluster: + name: Stop buildx cluster + needs: + - start_cluster + - publish_docker + runs-on: ubuntu-latest + if: ${{ github.repository_owner == 'hyperledger' && always() }} + permissions: + id-token: write + contents: read + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-region: ${{ secrets.AWS_BUILDX_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + + - uses: actions/checkout@v2 + + - name: Destroy cluster + uses: ./.github/actions/ec2-runners + with: + action: stop + label: ${{ needs.start_cluster.outputs.label }} + + - name: Notify Slack of Failure + if: cancelled() || failure() + uses: 8398a7/action-slack@v3 + with: + status: ${{ job.status }} + fields: repo,message,author,job + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + publish_to_crates: needs: unit_test_grid if: >-