cloudbuild.yaml

---
# Copyright 2017 Google LLC All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Google Cloud Builder CI configuration
#
steps:
  #
  # Cancel the previous build for the same branch
  #
  - name: gcr.io/cloud-builders/gcloud-slim:latest
    id: cancelot
    entrypoint: bash
    args: [./ci/cancelot.sh, --current_build_id, $BUILD_ID]

  #
  # Print Docker version
  #
  - name: gcr.io/cloud-builders/docker:24.0.6
    id: docker-version
    args: [--version]

  #
  # Restore any caches
  #
  - name: us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache
    id: htmltest-restore-cache
    args: [--bucket=gs://$_CACHE_BUCKET, --key=$_HTMLTEST_CACHE_KEY]
    waitFor: ['-']
  - name: us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache
    id: cpp-sdk-build-restore-cache
    args:
      - --bucket=gs://$_CACHE_BUCKET
      - --key=$_CPP_SDK_BUILD_CACHE_KEY-$( checksum sdks/cpp/CMakeLists.txt )
    waitFor: ['-']
  - name: us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache
    id: cpp-sdk-conformance-restore-cache
    args:
      - --bucket=gs://$_CACHE_BUCKET
      - --key=$_CPP_SDK_CONFORMANCE_CACHE_KEY-$( checksum sdks/cpp/CMakeLists.txt )
    waitFor: ['-']
  - name: us-docker.pkg.dev/$PROJECT_ID/ci/restore_cache
    id: rust-sdk-build-restore-cache
    args:
      - --bucket=gs://$_CACHE_BUCKET
      - --key=$_RUST_SDK_BUILD_CACHE_KEY-$( checksum test/sdk/rust/Cargo.toml )
    waitFor: ['-']

  #
  # Creates the initial make + docker build platform
  #
  - name: ubuntu
    args:
      - bash
      - -c
      - "echo 'FROM gcr.io/cloud-builders/docker:24.0.6\nRUN apt-get install make\nENTRYPOINT\
        \ [\"/usr/bin/make\"]' > Dockerfile.build"
    waitFor: ['-']
  - name: gcr.io/cloud-builders/docker:24.0.6
    id: build-make-docker
    args: [build, -f, Dockerfile.build, -t, make-docker, .]  # we need docker and make to run everything.

  #
  # pull the main build image if it exists
  #
  - name: make-docker
    id: pull-build-image
    dir: build
    env: ['REGISTRY=${_REGISTRY}']
    args: [pull-build-image]
    waitFor:
      - build-make-docker

  #
  # pull the sdk base image, if it exists
  #
  - name: make-docker
    id: pull-build-sdk-base-image
    dir: build
    env: ['REGISTRY=${_REGISTRY}']
    args: [pull-build-sdk-base-image]
    waitFor:
      - build-make-docker

  #
  # Ensure that there is the sdk base image, so that both tests
  # and build can use them
  #
  - name: make-docker
    id: ensure-build-sdk-image-base
    waitFor: [pull-build-sdk-base-image]
    dir: build
    args: [ensure-build-sdk-image-base]

  #
  # Preventing SDKs failure in CI
  #
  - name: make-docker
    id: test-gen-all-sdk-grpc
    waitFor: [ensure-build-sdk-image-base]
    dir: build
    args: [test-gen-all-sdk-grpc]

  #
  # Preventing Broken PR Merges in CI
  #
  - name: make-docker
    id: test-gen-crd-code
    waitFor: [pull-build-image]
    dir: build
    args: [test-gen-crd-code]

  #
  # Runs the linter -- but also builds the build image, if not able to download
  #
  - name: make-docker
    id: lint
    waitFor:
      - pull-build-image
      - test-gen-crd-code
      - test-gen-all-sdk-grpc
    dir: build
    args: [lint]  # pull the build image if it exists

  #
  # Push the build image, can run while we do other things.
  #
  - name: make-docker
    waitFor: [lint]
    dir: build
    env: ['REGISTRY=${_REGISTRY}']
    args: [push-build-image]  # push the build image (which won't do anything if it's already there)

  #
  # Push the sdk base build image, which can also run while we do other things
  #
  - name: make-docker
    waitFor: [ensure-build-sdk-image-base]
    dir: build
    env: ['REGISTRY=${_REGISTRY}']
    args: [push-build-sdk-base-image]

  #
  # Build all the images and sdks, and push them up to the repository
  #
  - name: make-docker
    id: build-images
    waitFor: [lint]
    dir: build
    env: ['REGISTRY=${_REGISTRY}']
    args: [-j, '4', build-images]
  - name: make-docker
    id: push-images
    waitFor: [build-images]
    dir: build
    env: ['REGISTRY=${_REGISTRY}']
    args: [-j, '4', push]
  - name: make-docker
    id: build-sdks
    waitFor:
      - lint
      - cpp-sdk-build-restore-cache
      - ensure-build-sdk-image-base
    dir: build
    args: [-j, '4', --output-sync=recurse, build-sdks]

  #
  # Example version checks
  #
  - name: make-docker
    id: check-example-versions
    dir: build
    args: [check-example-versions]
    waitFor:
      - push-images

  #
  # Run the all the automated tests (except e2e) in parallel
  #
  - name: make-docker
    id: tests
    waitFor:
      - lint
      - ensure-build-sdk-image-base
      - htmltest-restore-cache
      - build-sdks
    dir: build
    args: [-j, '5', --output-sync=target, test]

  #
  # SDK conformance tests
  #
  - name: make-docker
    id: sdk-conformance
    dir: build
    env: ['REGISTRY=${_REGISTRY}']
    args: [-j, '5', --output-sync=target, run-sdk-conformance-tests]
    waitFor:
      - build-images
      - tests

  #
  # Site preview
  #
  - name: make-docker  # build a preview of the website
    id: site-static
    waitFor: [tests]
    dir: build
    args: [site-static-preview, site-gen-app-yaml, SERVICE=preview]
  # deploy the preview of the website; # don't promote, as it can cause failures
  # when two deploys try and promote at the same time.
  - name: gcr.io/cloud-builders/gcloud
    id: deploy-site-static
    waitFor: [site-static]
    dir: site
    args: [app, deploy, .app.yaml, --no-promote, --version=$SHORT_SHA]
    env:
      - GOPATH=/workspace/go
      - GO111MODULE=on

  #
  # End to end tests
  #

  # Build and Push upgrade test
  - name: make-docker
    id: push-upgrade-test
    dir: test/upgrade
    env: ['REGISTRY=${_REGISTRY}']
    args: [push]
    waitFor:
      - push-images

  # Wait for us to be the oldest ongoing build before we run upgrade and e2e tests
  - name: gcr.io/google.com/cloudsdktool/cloud-sdk
    id: wait-to-become-leader
    waitFor: [push-images]
    script: |
      #!/usr/bin/env bash
      TS='date --utc +%FT%TZ'  # e.g. 2023-01-26T13:30:37Z
      echo "$(${TS}): Waiting to become oldest running build"
      while true; do
        # Filter to running builds with tag 'ci', which covers any builds running e2es.
        BUILD_FILTER="status=WORKING AND tags='ci'"
        OLDEST=$(gcloud builds list --filter "${BUILD_FILTER}" --format="value(id,startTime)" --sort-by=startTime --limit=1)
        echo "$(${TS}): Oldest is (id startTime): ${OLDEST}"
        if echo ${OLDEST} | grep -q "${BUILD_ID}"; then
          echo "$(${TS}): That's us, we're done!"
          break
        fi
        sleep 60
      done
    timeout: 10800s  # 3h - if you change this, change the global timeout as well
    env:
      - CLOUDSDK_CORE_PROJECT=$PROJECT_ID
      - BUILD_ID=$BUILD_ID
      - TRIGGER_NAME=$TRIGGER_NAME

  # Run the upgrade tests parallel, fail this step if any of the tests fail
  - name: gcr.io/google.com/cloudsdktool/cloud-sdk
    id: submit-upgrade-test-cloud-build
    dir: test/upgrade
    entrypoint: bash
    args:
      - -c
      - |
        #!/usr/bin/env bash
        set -e
        set -o pipefail
        export KUBECONFIG="/root/.kube/config"
        mkdir -p /go/src/agones.dev/ /root/.kube/
        ln -s /workspace /go/src/agones.dev/agones
        cd /go/src/agones.dev/agones/test/upgrade

        pids=()
        typeset -A waitPids    # Associative array for mapping `kubectl wait job` pid -> `kubectl wait job` output log name
        tmpdir=$(mktemp -d)
        trap 'rm -rf -- "$tmpdir"' EXIT SIGTERM

        # Update image tags to include the current build version.
        DevVersion="${_BASE_VERSION}-dev-$(git rev-parse --short=7 HEAD)"
        export DevVersion
        sed "s/\${DevVersion}/${DevVersion}/" upgradeTest.yaml > "${tmpdir}"/upgradeTest.yaml
        sed "s/\${DevVersion}/${DevVersion}/" versionMap.yaml > "${tmpdir}"/versionMap.yaml

        # Kill all currently running child processes on exit or if a non-zero signal is seen
        trap 'echo Cleaning up any remaining running pids: $(jobs -p) ; kill $(jobs -p) 2> /dev/null || :' EXIT SIGTERM

        cloudProducts=("generic" "gke-autopilot")
        declare -A versionsAndRegions=( [1.31]=us-east1 [1.30]=us-central1 [1.29]=us-west1 )

        for cloudProduct in "${cloudProducts[@]}"
        do
          for version in "${!versionsAndRegions[@]}"
          do
            region=${versionsAndRegions[$version]}
            if [ "$cloudProduct" = generic ]
            then
              testCluster="standard-upgrade-test-cluster-${version//./-}"
            else
              testCluster="gke-autopilot-upgrade-test-cluster-${version//./-}"
            fi
            testClusterLocation="${region}"

            gcloud container clusters get-credentials "$testCluster" --region="$testClusterLocation" --project="$PROJECT_ID"

            if [ "$cloudProduct" = gke-autopilot ] ; then
              # For autopilot clusters use evictable "balloon" pods to keep a buffer in node pool autoscaling.
              kubectl apply -f evictablePods.yaml
            fi

            # Clean up any existing job / namespace / apiservice from previous run
            echo Checking if resources from a previous build of upgrade-test-runner exist and need to be cleaned up on cluster "${testCluster}".
            if kubectl get jobs | grep upgrade-test-runner ; then
              echo Deleting job from previous run of upgrade-test-runner on cluster "${testCluster}".
              kubectl delete job upgrade-test-runner
              kubectl wait --for=delete pod -l job-name=upgrade-test-runner --timeout=5m
            fi

            # Check if there are any dangling game servers.
            if kubectl get gs | grep ".*"; then
              # Remove any finalizers so that dangling game servers can be manually deleted.
              kubectl get gs -o=custom-columns=:.metadata.name --no-headers | xargs kubectl patch gs -p '{"metadata":{"finalizers":[]}}' --type=merge
              sleep 5
              echo Deleting game servers from previous run of upgrade-test-runner on cluster "${testCluster}".
              kubectl delete gs -l app=sdk-client-test
            fi

            if kubectl get po -l app=sdk-client-test | grep ".*"; then
              echo Deleting pods from previous run of upgrade-test-runner on cluster "${testCluster}".
              kubectl delete po -l app=sdk-client-test
              kubectl wait --for=delete pod -l app=sdk-client-test --timeout=5m
            fi

            # The v1.allocation.agones.dev apiservice does not get removed automatically and will prevent the namespace from terminating.
            if kubectl get apiservice | grep v1.allocation.agones.dev ; then
                echo Deleting v1.allocation.agones.dev from previous run of upgrade-test-runner on cluster "${testCluster}".
                kubectl delete apiservice v1.allocation.agones.dev
            fi

            if kubectl get namespace | grep agones-system ; then
              echo Deleting agones-system namespace from previous run of upgrade-test-runner on cluster "${testCluster}".
              kubectl delete namespace agones-system
              kubectl wait --for=delete ns agones-system --timeout=5m
            fi

            if kubectl get crds | grep agones ; then
              echo Deleting crds from previous run of upgrade-test-runner on cluster "${testCluster}".
              kubectl get crds -o=custom-columns=:.metadata.name | grep agones | xargs kubectl delete crd
            fi

            echo kubectl apply -f permissions.yaml on cluster "${testCluster}"
            kubectl apply -f permissions.yaml
            echo kubectl apply -f versionMap.yaml on cluster "${testCluster}"
            kubectl apply -f "${tmpdir}"/versionMap.yaml
            echo kubectl apply -f gameserverTemplate.yaml on cluster "${testCluster}"
            kubectl apply -f gameserverTemplate.yaml

            echo kubectl apply -f upgradeTest.yaml on cluster "${testCluster}"
            kubectl apply -f "${tmpdir}"/upgradeTest.yaml

            # We need to wait for job pod to be created and ready before we can wait on the job itself.
            # TODO: Once all test clusters are at Kubernetes Version >= 1.31 use `kubectl wait --for=create` instead of sleep.
            # kubectl wait --for=create pod -l job-name=upgrade-test-runner --timeout=1m
            sleep 10s
            kubectl wait --for=condition=ready pod -l job-name=upgrade-test-runner --timeout=5m

            echo Wait for job upgrade-test-runner to complete or fail on cluster "${testCluster}"
            kubectl wait job/upgrade-test-runner --timeout=20m --for jsonpath='{.status.conditions[*].status}'=True -o jsonpath='{.status.conditions[*].type}' | tee "${tmpdir}"/"${testCluster}".log &
            waitPid=$!
            pids+=( "$waitPid" )
            waitPids[$waitPid]="${tmpdir}"/"${testCluster}".log
          done
        done

        for pid in "${pids[@]}"; do
          # This block executes when the process exits and pid status==0
          if wait $pid; then
            outputLog="${waitPids[$pid]}"
            # wait for output to finish writing to file
            until [ -s "$outputLog" ]; do sleep 1; done
            output=$(<"${outputLog}")
            echo  "${outputLog}": "${output}"

            # "Complete" is successful job run.
            # Version 1.31 has "SuccessCriteriaMet" as the first completion status returned, or "FailureTarget" in case of failure.
            if [ "$output" == "Complete" ] || [ "$output" == "SuccessCriteriaMet" ] ; then
              continue
            else
              exit 1
            fi
          # This block executes when the process exits and pid status!=0
          else
            status=$?
            outputLog="${waitPids[$pid]}"
            echo "One of the upgrade tests pid $pid from cluster log $outputLog exited with a non-zero status ${status}."
            exit $status
          fi
        done
        echo "End of Upgrade Tests"

    waitFor:
      - wait-to-become-leader
      - push-upgrade-test

  # cancel all the orphan e2e test cloud builds, fail to cancel any of the build will fail this whole build
  - name: gcr.io/cloud-builders/gcloud
    id: cancel-orphan-e2e-tests
    waitFor: [wait-to-become-leader]
    script: |
      #!/usr/bin/env bash
      until gcloud builds list --ongoing --filter "tags:'e2e-test'" --format="value(id)" | xargs --no-run-if-empty gcloud builds cancel
      do
        echo "== encountered error; assuming retryable error and trying again =="
        sleep 5
      done

  # kick off the child e2e test cloud builds in parallel, fail this build if any of the child build fails
  - name: gcr.io/cloud-builders/gcloud
    id: submit-e2e-test-cloud-build
    entrypoint: bash
    args:
      - -c
      - |
        set -e
        set -o pipefail
        pids=()
        cloudProducts=("generic" "gke-autopilot")
        declare -A versionsAndRegions=( [1.29]=europe-west1 [1.30]=asia-east1 [1.31]=us-east1 )

        # Keep in sync with (the inverse of) pkg/util/runtime/features.go:featureDefaults
        featureWithGate="PlayerAllocationFilter=true&PlayerTracking=true&CountsAndLists=false&RollingUpdateFix=true&PortRanges=true&PortPolicyNone=true&ScheduledAutoscaler=true&DisableResyncOnSDKServer=false&AutopilotPassthroughPort=false&GKEAutopilotExtendedDurationPods=false&Example=true"
        featureWithoutGate=""

        # Use this if specific feature gates can only be supported on specific Kubernetes versions.
        declare -A featureWithGateByVersion=( [1.29]="${featureWithGate}" [1.30]="${featureWithGate}" [1.31]="${featureWithGate}")

        for cloudProduct in ${cloudProducts[@]}
        do
          for version in "${!versionsAndRegions[@]}"
          do
            withGate=${featureWithGateByVersion[$version]}
            region=${versionsAndRegions[$version]}
            if [ $cloudProduct = generic ]
            then
              testCluster="standard-e2e-test-cluster-${version//./-}"
            else
              testCluster="gke-autopilot-e2e-test-cluster-${version//./-}"
            fi
            testClusterLocation="${region}"
            { stdbuf -oL -eL gcloud builds submit . --suppress-logs --config=./ci/e2e-test-cloudbuild.yaml \
              --substitutions _FEATURE_WITH_GATE=$withGate,_FEATURE_WITHOUT_GATE=$featureWithoutGate,_CLOUD_PRODUCT=$cloudProduct,_TEST_CLUSTER_NAME=$testCluster,_TEST_CLUSTER_LOCATION=$testClusterLocation,_REGISTRY=${_REGISTRY},_PARENT_COMMIT_SHA=${COMMIT_SHA},_PARENT_BUILD_ID=${BUILD_ID} \
              |& stdbuf -i0 -oL -eL grep -v " tarball " \
              |& stdbuf -i0 -oL -eL sed "s/^/${cloudProduct}-${version}: /"; } &
            pids+=($!)
          done
        done

        # If any of the subprocess exit with nonzero code, exit the main process and kill all subprocesses
        for pid in "${pids[@]}"; do
          if wait -n; then
            :
          else
            status=$?
            echo "One of the e2e test child cloud build exited with nonzero status $status. Aborting."
            for pid in "${pids[@]}"; do
              # Send a termination signal to all the children, and ignore errors
              # due to children that no longer exist.
              kill "$pid" 2> /dev/null || :
              echo "killed $pid"
            done
            exit "$status"
          fi
        done
        echo "all done"
    waitFor:
      - cancel-orphan-e2e-tests

  #
  # Store caches, to speed up subsequent builds (considerably)
  #
  - name: us-docker.pkg.dev/$PROJECT_ID/ci/save_cache
    args:
      - --bucket=gs://$_CACHE_BUCKET
      - --key=$_CPP_SDK_BUILD_CACHE_KEY-$( checksum sdks/cpp/CMakeLists.txt )
      - --path=sdks/cpp/.build  # CPP SDK build
    id: cpp-sdk-build-save-cache
    waitFor:
      - build-sdks
  - name: us-docker.pkg.dev/$PROJECT_ID/ci/save_cache
    args:
      - --bucket=gs://$_CACHE_BUCKET
      - --key=$_CPP_SDK_CONFORMANCE_CACHE_KEY-$( sdks/cpp/CMakeLists.txt )
      - --path=test/sdk/cpp/sdk  # CPP conformance test build
    id: cpp-sdk-conformance-save-cache
    waitFor:
      - sdk-conformance
  - name: us-docker.pkg.dev/$PROJECT_ID/ci/save_cache
    args:
      - --bucket=gs://$_CACHE_BUCKET
      - --key=$_RUST_SDK_BUILD_CACHE_KEY-$( checksum test/sdk/rust/Cargo.toml )
      - --path=test/sdk/rust/.cargo  # Rust conformance test build
      - --path=test/sdk/rust/.cargo-targets
      - --no-clobber
    id: rust-build-save-cache
    waitFor:
      - sdk-conformance
  - name: us-docker.pkg.dev/$PROJECT_ID/ci/save_cache
    args:
      - --bucket=gs://$_CACHE_BUCKET
      - --key=$_HTMLTEST_CACHE_KEY
      - --path=site/tmp  # htmltest URL checks
    id: htmltest-save-cache
    waitFor:
      - tests

  #
  # Zip up artifacts and push to storage
  #
  - name: gcr.io/cloud-builders/gsutil
    waitFor: [build-sdks, tests]
    dir: sdks/cpp/.archives
    args: [-m, cp, '*.tar.gz', gs://agones-artifacts/cpp-sdk]
  - name: gcr.io/cloud-builders/gsutil
    waitFor: [build-images, tests]
    dir: cmd/sdk-server/bin
    args: [-m, cp, '*.zip', gs://agones-artifacts/sdk-server]

  #
  # Cleanup services "preview", "development", "default"
  #
  - name: gcr.io/cloud-builders/gcloud
    id: cleanup-services
    waitFor: [wait-to-become-leader]
    allowFailure: true
    entrypoint: bash
    args:
      - -c
      - |
        set -e
        set -o pipefail
        for service in preview development default; do
          echo "Cleaning up $service service..."
          gcloud app versions list --service=$service --filter="traffic_split<1" --sort-by=~last_deployed_time.datetime --format="table[no-heading](version.id)" | tail -n +50 | xargs --no-run-if-empty gcloud app versions delete --service=$service --quiet
        done

substitutions:
  _BASE_VERSION: 1.46.0
  _CACHE_BUCKET: agones-build-cache
  _HTMLTEST_CACHE_KEY: htmltest-0.10.1
  _CPP_SDK_BUILD_CACHE_KEY: cpp-sdk-build
  _CPP_SDK_CONFORMANCE_CACHE_KEY: cpp-sdk-conformance
  _RUST_SDK_BUILD_CACHE_KEY: rust-sdk-build
  _REGISTRY: us-docker.pkg.dev/${PROJECT_ID}/ci
tags: [ci, 'commit-${COMMIT_SHA}']
timeout: 18000s  # 5h: 3h (wait-to-become-leader) + 1.5h (e2e timeout) + 0.5h (everything else)
queueTtl: 259200s  # 72h
images:
  - ${_REGISTRY}/agones-controller
  - ${_REGISTRY}/agones-extensions
  - ${_REGISTRY}/agones-sdk
  - ${_REGISTRY}/agones-ping
  - ${_REGISTRY}/agones-allocator
logsBucket: gs://agones-build-logs
options:
  machineType: E2_HIGHCPU_32
  dynamic_substitutions: true