From 15b01cf91751222b180c2531c833157e4d7d2f5a Mon Sep 17 00:00:00 2001 From: Mike Bond Date: Thu, 16 Jan 2025 22:07:58 -0800 Subject: [PATCH 1/9] Temporarily remove agent-cleanser steps since the Xamarin AzDev extension has mistakenly been uninstalled --- tools/devops/automation/templates/common/setup.yml | 6 +++--- tools/devops/automation/templates/mac/build.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/devops/automation/templates/common/setup.yml b/tools/devops/automation/templates/common/setup.yml index 7dcd48e1b2d9..7ebf25b0bc2e 100644 --- a/tools/devops/automation/templates/common/setup.yml +++ b/tools/devops/automation/templates/common/setup.yml @@ -6,9 +6,9 @@ parameters: type: string steps: -- template: agent-cleanser/v1.yml@yaml-templates # Uninstalls mono, Xamarin.Mac (if installed) plus cleanses the Provisionator Xcode cache and kills processes at the end - parameters: - CleanseProcesses: true +#- template: agent-cleanser/v1.yml@yaml-templates # Uninstalls mono, Xamarin.Mac (if installed) plus cleanses the Provisionator Xcode cache and kills processes at the end +# parameters: +# CleanseProcesses: true - bash: $(Build.SourcesDirectory)/$(BUILD_REPOSITORY_TITLE)/tools/devops/automation/scripts/disable-codeql-arm64.sh displayName: 'Disable CodeQL on arm64' diff --git a/tools/devops/automation/templates/mac/build.yml b/tools/devops/automation/templates/mac/build.yml index cfe43a84a47c..c650abf28e01 100644 --- a/tools/devops/automation/templates/mac/build.yml +++ b/tools/devops/automation/templates/mac/build.yml @@ -39,9 +39,9 @@ steps: repositoryAlias: ${{ parameters.repositoryAlias }} commit: ${{ parameters.commit }} -- template: agent-cleanser/v1.yml@yaml-templates # Uninstalls mono, Xamarin.Mac (if installed) plus cleanses the Provisionator Xcode cache and kills processes at the end - parameters: - CleanseProcesses: true +#- template: agent-cleanser/v1.yml@yaml-templates # Uninstalls mono, Xamarin.Mac (if installed) plus cleanses the Provisionator Xcode cache and kills processes at the end +# parameters: +# CleanseProcesses: true - bash: $(Build.SourcesDirectory)/$(BUILD_REPOSITORY_TITLE)/tools/devops/automation/scripts/disable-codeql-arm64.sh displayName: 'Disable CodeQL on arm64' From a298ca624b438486732d7d9a58ce1b50b8a57469 Mon Sep 17 00:00:00 2001 From: Mike Bond Date: Thu, 16 Jan 2025 23:44:19 -0800 Subject: [PATCH 2/9] Use yaml-templates that temporarily disable use of Xamarin AzDev tasks --- tools/devops/automation/build-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/devops/automation/build-pipeline.yml b/tools/devops/automation/build-pipeline.yml index de5ac8964149..10290ffc9571 100644 --- a/tools/devops/automation/build-pipeline.yml +++ b/tools/devops/automation/build-pipeline.yml @@ -126,7 +126,7 @@ resources: - repository: yaml-templates type: github name: xamarin/yaml-templates - ref: refs/heads/main + ref: refs/heads/dev/bond/disable-tasks # UNDONE: DO NOT MERGE TO MAIN endpoint: xamarin - repository: sdk-insertions From efd19742b5120adf406429571d532732135e3cea Mon Sep 17 00:00:00 2001 From: Mike Bond Date: Thu, 16 Jan 2025 23:56:43 -0800 Subject: [PATCH 3/9] build-pull-request: Use yaml-templates that disables use of Xamarin AzDev extension tasks --- tools/devops/automation/build-pull-request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/devops/automation/build-pull-request.yml b/tools/devops/automation/build-pull-request.yml index ee0b940d0a0c..37e1460747a1 100644 --- a/tools/devops/automation/build-pull-request.yml +++ b/tools/devops/automation/build-pull-request.yml @@ -117,7 +117,7 @@ resources: - repository: yaml-templates type: github name: xamarin/yaml-templates - ref: refs/heads/main + ref: refs/heads/dev/bond/disable-tasks # UNDONE: DO NOT MERGE TO MAIN endpoint: xamarin - repository: sdk-insertions From c5cfb504047e6d9b97a61e32428f12919aa8d88a Mon Sep 17 00:00:00 2001 From: Mike Bond Date: Fri, 17 Jan 2025 00:12:06 -0800 Subject: [PATCH 4/9] Replace Provisionator task with provisionator-bootstrap script --- bot-provisioning/provisionator-bootstrap.sh | 84 +++++++++++++++++++ .../automation/templates/build/build.yml | 19 +++++ 2 files changed, 103 insertions(+) create mode 100644 bot-provisioning/provisionator-bootstrap.sh diff --git a/bot-provisioning/provisionator-bootstrap.sh b/bot-provisioning/provisionator-bootstrap.sh new file mode 100644 index 000000000000..07978be06e9f --- /dev/null +++ b/bot-provisioning/provisionator-bootstrap.sh @@ -0,0 +1,84 @@ +#!/bin/bash -e +# +# Remove the following GUID if you do not wish for this script self-update: +# D6F71FB5-F2A7-4A62-86D3-10DFE08301CC +# https://github.com/xamarin/provisionator + +while getopts ":t:" opt; do + case "${opt}" in + t) + TOKEN=${OPTARG} + shift 2 + ;; + esac +done + +function selfdir { (cd "$(dirname "$1")"; echo "$PWD"; ) } + +selfdir=$(selfdir "$0") + +channel="${PROVISIONATOR_CHANNEL:-latest}" + +# Grab auth token from Environment first +# Otherwise set to provided option, if available +base_url="https://dl.internalx.com/provisionator/664bd334021e3102cdef1af66c4fc9f1b2ecd2a21b47419e80d08da1f6c61c2a/${channel}" +auth_token=$AUTH_TOKEN_GITHUB_COM +if [ -z $auth_token ]; then + auth_token=$TOKEN +fi +if [ -z $auth_token ]; then + echo "WARNING: Please set AUTH_TOKEN_GITHUB_COM to a GitHub PAT before running provisionator or run provisionator.sh -t ..." + echo "At a future point, Provisionator will not be able to bootstrap without the PAT set properly" + base_url="https://bosstoragemirror.blob.core.windows.net/provisionator/664bd334021e3102cdef1af66c4fc9f1b2ecd2a21b47419e80d08da1f6c61c2a/${channel}" +fi + +latest_version_url="${base_url}/version" + +archive_name="provisionator.osx.10.11-x64.zip" +archive_path="${selfdir}/${archive_name}" +archive_extract_path="${selfdir}/_provisionator" +archive_url="${base_url}/${archive_name}" +binary_path="${archive_extract_path}/provisionator" + +set +e +if [ -z $auth_token ]; then + latest_version="$(curl -fsL "${latest_version_url}")" +else + latest_version="$(curl -fsL -H "Authorization: token ${auth_token}" "${latest_version_url}")" +fi +if [ $? != 0 ]; then + echo "Unable to determine latest version from ${latest_version_url}" + echo "Please check to make sure a valid GitHub PAT was provided" + exit 1 +fi +set -e + +function update_in_place { + echo "Downloading Provisionator $latest_version..." + local progress_type="-s" + tty -s && progress_type="-#" + if [ -z $auth_token ]; then + curl -fL $progress_type -o "$archive_path" "$archive_url" + else + curl -fLH "Authorization: token ${auth_token}" $progress_type -o "$archive_path" "$archive_url" + fi + rm -rf "$archive_extract_path" + unzip -q -o -d "$archive_extract_path" "$archive_path" + rm -f "$archive_path" +} + +if [ -f "$binary_path" ]; then + chmod +x "$binary_path" + current_version="$("$binary_path" -version 2>&1 || true)" + if [ "$latest_version" != "$current_version" ]; then + update_in_place + fi +else + update_in_place +fi + +# Opt out of macOS backward compat so correct Big Sur OS versions can be obtained. +# More info https://github.com/ziglang/zig/issues/7569 +export SYSTEM_VERSION_COMPAT=0 + +exec caffeinate "$binary_path" "$@" diff --git a/tools/devops/automation/templates/build/build.yml b/tools/devops/automation/templates/build/build.yml index bf9cc5259e9c..02bdc4d561b3 100644 --- a/tools/devops/automation/templates/build/build.yml +++ b/tools/devops/automation/templates/build/build.yml @@ -76,6 +76,25 @@ steps: github_token: ${{ parameters.gitHubToken }} timeoutInMinutes: 250 retryCountOnTaskFailure: ${{ parameters.retryCount }} # mono does give issues sometimes to download, we will retry + enabled: false # UNDONE: Use below script instead to provision until + + - bash: | + echo "PROVISIONATOR_CHANNEL: ${PROVISIONATOR_CHANNEL}" + githubToken="${{ parameters.gitHubToken }}" + if [[ -n $githubToken ]]; then + echo "Executing provisionator" + ./provisionator-bootstrap.sh $(Build.SourcesDirectory)/$(BUILD_REPOSITORY_TITLE)/tools/devops/build-provisioning.csx -v -v -v -v || exit 2 + xcode-select -p + xcodebuild -version + else + echo "ERROR: GitHub.Token build variable not set" + exit 1 + fi + env: + AUTH_TOKEN_GITHUB_COM: ${{ parameters.gitHubToken }} # Avoid having Provisionator retrieve the token from the keychain, which can lead to hangs on certain build agents; instead Provisionator will obtain the GitHub token from this environment variable + workingDirectory: bot-provisioning + timeoutInMinutes: 250 + displayName: Provision # Use the env variables that were set by the label parsing in the configure step # print some useful logging to allow to know what is going on AND allow make some From 328d54ef63282ef47dd7b64dbd2a5c5d4bb6a2ce Mon Sep 17 00:00:00 2001 From: Mike Bond Date: Fri, 17 Jan 2025 01:03:09 -0800 Subject: [PATCH 5/9] Provisionator script: Fix working directory to include xamarin-macios since multiple repos are being checked out --- tools/devops/automation/templates/build/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/devops/automation/templates/build/build.yml b/tools/devops/automation/templates/build/build.yml index 02bdc4d561b3..827b1272ca01 100644 --- a/tools/devops/automation/templates/build/build.yml +++ b/tools/devops/automation/templates/build/build.yml @@ -92,7 +92,7 @@ steps: fi env: AUTH_TOKEN_GITHUB_COM: ${{ parameters.gitHubToken }} # Avoid having Provisionator retrieve the token from the keychain, which can lead to hangs on certain build agents; instead Provisionator will obtain the GitHub token from this environment variable - workingDirectory: bot-provisioning + workingDirectory: $(Build.Sources)/xamarin-macios/bot-provisioning timeoutInMinutes: 250 displayName: Provision From a851eba174ab7df48c53ce1904d8ece2c4067334 Mon Sep 17 00:00:00 2001 From: Mike Bond Date: Fri, 17 Jan 2025 01:10:13 -0800 Subject: [PATCH 6/9] Restore agent-cleanser tasks since that template has been updated to temporarily to disable Xamarin AzDev extension tasks --- tools/devops/automation/templates/common/setup.yml | 6 +++--- tools/devops/automation/templates/mac/build.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/devops/automation/templates/common/setup.yml b/tools/devops/automation/templates/common/setup.yml index 7ebf25b0bc2e..7dcd48e1b2d9 100644 --- a/tools/devops/automation/templates/common/setup.yml +++ b/tools/devops/automation/templates/common/setup.yml @@ -6,9 +6,9 @@ parameters: type: string steps: -#- template: agent-cleanser/v1.yml@yaml-templates # Uninstalls mono, Xamarin.Mac (if installed) plus cleanses the Provisionator Xcode cache and kills processes at the end -# parameters: -# CleanseProcesses: true +- template: agent-cleanser/v1.yml@yaml-templates # Uninstalls mono, Xamarin.Mac (if installed) plus cleanses the Provisionator Xcode cache and kills processes at the end + parameters: + CleanseProcesses: true - bash: $(Build.SourcesDirectory)/$(BUILD_REPOSITORY_TITLE)/tools/devops/automation/scripts/disable-codeql-arm64.sh displayName: 'Disable CodeQL on arm64' diff --git a/tools/devops/automation/templates/mac/build.yml b/tools/devops/automation/templates/mac/build.yml index c650abf28e01..cfe43a84a47c 100644 --- a/tools/devops/automation/templates/mac/build.yml +++ b/tools/devops/automation/templates/mac/build.yml @@ -39,9 +39,9 @@ steps: repositoryAlias: ${{ parameters.repositoryAlias }} commit: ${{ parameters.commit }} -#- template: agent-cleanser/v1.yml@yaml-templates # Uninstalls mono, Xamarin.Mac (if installed) plus cleanses the Provisionator Xcode cache and kills processes at the end -# parameters: -# CleanseProcesses: true +- template: agent-cleanser/v1.yml@yaml-templates # Uninstalls mono, Xamarin.Mac (if installed) plus cleanses the Provisionator Xcode cache and kills processes at the end + parameters: + CleanseProcesses: true - bash: $(Build.SourcesDirectory)/$(BUILD_REPOSITORY_TITLE)/tools/devops/automation/scripts/disable-codeql-arm64.sh displayName: 'Disable CodeQL on arm64' From 9a7037b82a0560d07d97d4af9051baf8dc5214c5 Mon Sep 17 00:00:00 2001 From: Mike Bond Date: Fri, 17 Jan 2025 02:07:27 -0800 Subject: [PATCH 7/9] Fix: Provisionator script working directory --- tools/devops/automation/templates/build/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/devops/automation/templates/build/build.yml b/tools/devops/automation/templates/build/build.yml index 827b1272ca01..6d588f358397 100644 --- a/tools/devops/automation/templates/build/build.yml +++ b/tools/devops/automation/templates/build/build.yml @@ -92,7 +92,7 @@ steps: fi env: AUTH_TOKEN_GITHUB_COM: ${{ parameters.gitHubToken }} # Avoid having Provisionator retrieve the token from the keychain, which can lead to hangs on certain build agents; instead Provisionator will obtain the GitHub token from this environment variable - workingDirectory: $(Build.Sources)/xamarin-macios/bot-provisioning + workingDirectory: xamarin-macios/bot-provisioning timeoutInMinutes: 250 displayName: Provision From 0e33cad070e33201b7033077ffb70164a93e3665 Mon Sep 17 00:00:00 2001 From: Mike Bond Date: Fri, 17 Jan 2025 02:46:27 -0800 Subject: [PATCH 8/9] Make the provisionator bootstrap script executable --- tools/devops/automation/templates/build/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/devops/automation/templates/build/build.yml b/tools/devops/automation/templates/build/build.yml index 6d588f358397..9d0d54972790 100644 --- a/tools/devops/automation/templates/build/build.yml +++ b/tools/devops/automation/templates/build/build.yml @@ -83,6 +83,7 @@ steps: githubToken="${{ parameters.gitHubToken }}" if [[ -n $githubToken ]]; then echo "Executing provisionator" + chmod +x ./provisionator-bootstrap.sh ./provisionator-bootstrap.sh $(Build.SourcesDirectory)/$(BUILD_REPOSITORY_TITLE)/tools/devops/build-provisioning.csx -v -v -v -v || exit 2 xcode-select -p xcodebuild -version From 88331bee96967cdaa6f2ecb88b9473d12436b2ac Mon Sep 17 00:00:00 2001 From: Mike Bond Date: Fri, 17 Jan 2025 02:50:27 -0800 Subject: [PATCH 9/9] Use yaml-templates main branch where the Xamarin AzDev extension tasks have been disabled --- tools/devops/automation/build-pipeline.yml | 2 +- tools/devops/automation/build-pull-request.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/devops/automation/build-pipeline.yml b/tools/devops/automation/build-pipeline.yml index 10290ffc9571..de5ac8964149 100644 --- a/tools/devops/automation/build-pipeline.yml +++ b/tools/devops/automation/build-pipeline.yml @@ -126,7 +126,7 @@ resources: - repository: yaml-templates type: github name: xamarin/yaml-templates - ref: refs/heads/dev/bond/disable-tasks # UNDONE: DO NOT MERGE TO MAIN + ref: refs/heads/main endpoint: xamarin - repository: sdk-insertions diff --git a/tools/devops/automation/build-pull-request.yml b/tools/devops/automation/build-pull-request.yml index 37e1460747a1..ee0b940d0a0c 100644 --- a/tools/devops/automation/build-pull-request.yml +++ b/tools/devops/automation/build-pull-request.yml @@ -117,7 +117,7 @@ resources: - repository: yaml-templates type: github name: xamarin/yaml-templates - ref: refs/heads/dev/bond/disable-tasks # UNDONE: DO NOT MERGE TO MAIN + ref: refs/heads/main endpoint: xamarin - repository: sdk-insertions