From a5e1a3e8490c1eae72fd44c229569f314cfdc1ed Mon Sep 17 00:00:00 2001 From: azure-sdk Date: Fri, 19 Apr 2024 17:33:24 +0000 Subject: [PATCH] Sync eng/common directory with azure-sdk-tools repository --- eng/common/InterdependencyGraph.html | 356 +++ eng/common/README.md | 3 + .../TestResources/New-TestResources.cmd | 17 + .../TestResources/New-TestResources.ps1 | 1122 +++++++ .../TestResources/New-TestResources.ps1.md | 727 +++++ eng/common/TestResources/README.md | 192 ++ .../TestResources/Remove-TestResources.cmd | 17 + .../TestResources/Remove-TestResources.ps1 | 324 ++ .../TestResources/Remove-TestResources.ps1.md | 308 ++ .../TestResources/SubConfig-Helpers.ps1 | 161 + .../TestResources/Update-TestResources.cmd | 17 + .../TestResources/Update-TestResources.ps1 | 204 ++ .../TestResources/Update-TestResources.ps1.md | 153 + .../build-test-resource-config.yml | 46 + .../TestResources/clouds/AzureChinaCloud.json | 15 + .../TestResources/clouds/AzureCloud.json | 9 + .../clouds/AzureUSGovernment.json | 15 + .../TestResources/deploy-test-resources.yml | 63 + .../TestResources/remove-test-resources.yml | 40 + .../TestResources/setup-environments.yml | 34 + .../docgeneration/Generate-DocIndex.ps1 | 189 ++ eng/common/docgeneration/assets/logo.svg | 76 + eng/common/docgeneration/docfx.json | 72 + .../matthews/partials/affix.tmpl.partial | 17 + .../partials/class.header.tmpl.partial | 100 + .../matthews/partials/class.tmpl.partial | 210 ++ .../matthews/partials/enum.tmpl.partial | 24 + .../matthews/partials/head.tmpl.partial | 28 + .../matthews/partials/namespace.tmpl.partial | 17 + .../templates/matthews/styles/main.css | 311 ++ .../templates/matthews/styles/main.js | 238 ++ eng/common/pipelines/codeowners-linter.yml | 58 + .../jobs/archetype-sdk-tests-generate.yml | 116 + .../pipelines/templates/jobs/docindex.yml | 67 + .../templates/jobs/generate-job-matrix.yml | 140 + eng/common/pipelines/templates/jobs/perf.yml | 181 ++ .../templates/jobs/prepare-pipelines.yml | 200 ++ .../stages/archetype-sdk-tool-pwsh.yml | 44 + .../templates/steps/bypass-local-dns.yml | 11 + .../templates/steps/cache-ps-modules.yml | 14 + .../templates/steps/check-spelling.yml | 34 + .../templates/steps/cosmos-emulator.yml | 16 + .../templates/steps/create-apireview.yml | 43 + .../templates/steps/create-pull-request.yml | 59 + .../steps/create-tags-and-git-release.yml | 25 + .../pipelines/templates/steps/credscan.yml | 55 + .../steps/daily-dev-build-variable.yml | 24 + .../templates/steps/detect-api-changes.yml | 28 + .../steps/devops-variables-clear.yml | 20 + .../templates/steps/devops-variables-set.yml | 21 + .../templates/steps/docker-pull-image.yml | 19 + .../steps/docsms-ensure-validation.yml | 12 + .../steps/enable-long-path-support.yml | 10 + .../steps/eng-common-workflow-enforcer.yml | 33 + .../templates/steps/git-push-changes.yml | 43 + .../steps/install-pipeline-generation.yml | 16 + .../templates/steps/mashup-doc-index.yml | 81 + .../pipelines/templates/steps/policheck.yml | 19 +- .../templates/steps/publish-1es-artifact.yml | 32 + .../templates/steps/publish-artifact.yml | 27 + .../templates/steps/publish-blobs.yml | 33 + .../steps/replace-relative-links.yml | 217 ++ .../pipelines/templates/steps/retain-run.yml | 21 + .../templates/steps/run-pester-tests.yml | 57 + .../steps/set-daily-docs-branch-name.yml | 14 + .../templates/steps/set-default-branch.yml | 16 + .../steps/set-test-pipeline-version.yml | 19 + .../templates/steps/sparse-checkout.yml | 125 + .../steps/update-docsms-metadata.yml | 112 + .../templates/steps/validate-all-packages.yml | 34 + .../templates/steps/validate-filename.yml | 18 + .../templates/steps/verify-agent-os.yml | 18 + .../templates/steps/verify-changelog.yml | 26 + .../templates/steps/verify-links.yml | 31 + .../templates/steps/verify-path-length.yml | 50 + .../templates/steps/verify-readme.yml | 29 + .../steps/verify-restapi-spec-location.yml | 24 + .../templates/steps/verify-samples.yml | 25 + .../steps/write-filesystemmetrics.yml | 7 + eng/common/scripts/Add-IssueComment.ps1 | 28 + eng/common/scripts/Add-IssueLabels.ps1 | 28 + eng/common/scripts/Add-RetentionLease.ps1 | 46 + eng/common/scripts/Cadl-Project-Generate.ps1 | 101 + eng/common/scripts/Cadl-Project-Sync.ps1 | 127 + eng/common/scripts/ChangeLog-Operations.ps1 | 450 +++ eng/common/scripts/Cosmos-Emulator.ps1 | 149 + eng/common/scripts/Create-APIReview.ps1 | 332 ++ eng/common/scripts/Delete-RemoteBranches.ps1 | 132 + eng/common/scripts/Delete-RemoteTag.ps1 | 33 + eng/common/scripts/Detect-Api-Changes.ps1 | 138 + eng/common/scripts/Generate-PR-Diff.ps1 | 51 + .../scripts/Get-BuildSourceDescription.ps1 | 24 + eng/common/scripts/Get-PullRequestCreator.ps1 | 28 + .../scripts/Helpers/ApiView-Helpers.ps1 | 118 + .../Helpers/CommandInvocation-Helpers.ps1 | 42 + .../Helpers/DevOps-WorkItem-Helpers.ps1 | 1030 ++++++ .../scripts/Helpers/Metadata-Helpers.ps1 | 96 + .../scripts/Helpers/PSModule-Helpers.ps1 | 125 + .../scripts/Helpers/Package-Helpers.ps1 | 47 + .../scripts/Helpers/Resource-Helpers.ps1 | 206 ++ ...ervice-Level-Readme-Automation-Helpers.ps1 | 129 + eng/common/scripts/Helpers/git-helpers.ps1 | 112 + .../scripts/Helpers/git-helpers.tests.ps1 | 57 + eng/common/scripts/Import-AzModules.ps1 | 8 + eng/common/scripts/Invoke-DevOpsAPI.ps1 | 176 ++ eng/common/scripts/Invoke-GitHubAPI.ps1 | 474 +++ eng/common/scripts/New-RegenerateMatrix.ps1 | 102 + eng/common/scripts/New-ReleaseAsset.ps1 | 62 + eng/common/scripts/Package-Properties.ps1 | 167 + eng/common/scripts/Prepare-Release.ps1 | 217 ++ eng/common/scripts/Queue-Pipeline.ps1 | 132 + .../scripts/Save-Package-Properties.ps1 | 142 + eng/common/scripts/SemVer.ps1 | 365 +++ .../Service-Level-Readme-Automation.ps1 | 130 + eng/common/scripts/SetTestPipelineVersion.ps1 | 56 + eng/common/scripts/Submit-PullRequest.ps1 | 141 + eng/common/scripts/Test-SampleMetadata.ps1 | 540 ++++ .../scripts/TypeSpec-Project-Generate.ps1 | 121 + .../scripts/TypeSpec-Project-Process.ps1 | 228 ++ eng/common/scripts/TypeSpec-Project-Sync.ps1 | 147 + eng/common/scripts/Update-ChangeLog.ps1 | 140 + .../Update-DevOps-Release-WorkItem.ps1 | 113 + eng/common/scripts/Update-DocsMsMetadata.ps1 | 229 ++ .../scripts/Update-DocsMsPackageMonikers.ps1 | 122 + eng/common/scripts/Update-DocsMsPackages.ps1 | 129 + eng/common/scripts/Update-DocsMsToc.ps1 | 318 ++ eng/common/scripts/Update-GeneratedSdks.ps1 | 16 + eng/common/scripts/Validate-All-Packages.ps1 | 52 + eng/common/scripts/Validate-Package.ps1 | 261 ++ eng/common/scripts/Verify-AgentOS.ps1 | 21 + eng/common/scripts/Verify-ChangeLog.ps1 | 30 + eng/common/scripts/Verify-Links.ps1 | 543 ++++ eng/common/scripts/Verify-Readme.ps1 | 105 + .../Verify-RequiredDocsJsonMembers.ps1 | 130 + eng/common/scripts/Verify-Resource-Ref.ps1 | 51 + .../scripts/Verify-RestApiSpecLocation.ps1 | 284 ++ .../scripts/Write-FileSystemMetrics.ps1 | 31 + eng/common/scripts/X509Certificate2/README.md | 24 + .../X509Certificate2/X509Certificate2.psm1 | 339 ++ .../scripts/artifact-metadata-parsing.ps1 | 207 ++ eng/common/scripts/check-for-git-changes.ps1 | 14 + .../check-spelling-in-changed-files.ps1 | 312 ++ eng/common/scripts/common.ps1 | 70 + .../scripts/copy-docs-to-blobstorage.ps1 | 252 ++ .../scripts/create-tags-and-git-release.ps1 | 42 + eng/common/scripts/get-changedfiles.ps1 | 40 + .../get-markdown-files-from-changed-files.ps1 | 13 + eng/common/scripts/git-branch-push.ps1 | 196 ++ .../scripts/job-matrix/Create-JobMatrix.ps1 | 45 + eng/common/scripts/job-matrix/README.md | 5 + .../job-matrix/job-matrix-functions.ps1 | 742 +++++ eng/common/scripts/logging.ps1 | 40 + .../stress-testing/deploy-stress-tests.ps1 | 43 + .../find-all-stress-packages.ps1 | 133 + .../generate-scenario-matrix.ps1 | 98 + .../stress-test-deployment-lib.ps1 | 505 +++ .../scripts/trust-proxy-certificate.ps1 | 10 + .../typespec/New-EmitterPackageJson.ps1 | 73 + .../typespec/New-EmitterPackageLock.ps1 | 58 + eng/common/spelling/Invoke-Cspell.ps1 | 194 ++ eng/common/spelling/package-lock.json | 2786 +++++++++++++++++ eng/common/spelling/package.json | 10 + eng/common/testproxy/apply-dev-cert.sh | 31 + eng/common/testproxy/docker-start-proxy.ps1 | 130 + eng/common/testproxy/dotnet-devcert.crt | 21 + eng/common/testproxy/dotnet-devcert.pfx | Bin 0 -> 2627 bytes eng/common/testproxy/localhost.conf | 23 + eng/common/testproxy/onboarding/README.md | 131 + .../onboarding/common-asset-functions.ps1 | 258 ++ .../onboarding/generate-assets-json.ps1 | 160 + eng/common/testproxy/publish-proxy-logs.yml | 19 + .../scripts/override-proxy-version.ps1 | 25 + .../scripts/resolve-asset-conflict/README.md | 62 + .../resolve-asset-conflict.ps1 | 79 + .../testproxy/scripts/tag-merge/README.md | 95 + .../scripts/tag-merge/merge-proxy-tags.ps1 | 304 ++ eng/common/testproxy/target_version.txt | 1 + eng/common/testproxy/test-proxy-docker.yml | 40 + .../testproxy/test-proxy-tool-shutdown.yml | 10 + eng/common/testproxy/test-proxy-tool.yml | 77 + 180 files changed, 24550 insertions(+), 8 deletions(-) create mode 100644 eng/common/InterdependencyGraph.html create mode 100644 eng/common/README.md create mode 100644 eng/common/TestResources/New-TestResources.cmd create mode 100644 eng/common/TestResources/New-TestResources.ps1 create mode 100644 eng/common/TestResources/New-TestResources.ps1.md create mode 100644 eng/common/TestResources/README.md create mode 100644 eng/common/TestResources/Remove-TestResources.cmd create mode 100644 eng/common/TestResources/Remove-TestResources.ps1 create mode 100644 eng/common/TestResources/Remove-TestResources.ps1.md create mode 100644 eng/common/TestResources/SubConfig-Helpers.ps1 create mode 100644 eng/common/TestResources/Update-TestResources.cmd create mode 100644 eng/common/TestResources/Update-TestResources.ps1 create mode 100644 eng/common/TestResources/Update-TestResources.ps1.md create mode 100644 eng/common/TestResources/build-test-resource-config.yml create mode 100644 eng/common/TestResources/clouds/AzureChinaCloud.json create mode 100644 eng/common/TestResources/clouds/AzureCloud.json create mode 100644 eng/common/TestResources/clouds/AzureUSGovernment.json create mode 100644 eng/common/TestResources/deploy-test-resources.yml create mode 100644 eng/common/TestResources/remove-test-resources.yml create mode 100644 eng/common/TestResources/setup-environments.yml create mode 100644 eng/common/docgeneration/Generate-DocIndex.ps1 create mode 100644 eng/common/docgeneration/assets/logo.svg create mode 100644 eng/common/docgeneration/docfx.json create mode 100644 eng/common/docgeneration/templates/matthews/partials/affix.tmpl.partial create mode 100644 eng/common/docgeneration/templates/matthews/partials/class.header.tmpl.partial create mode 100644 eng/common/docgeneration/templates/matthews/partials/class.tmpl.partial create mode 100644 eng/common/docgeneration/templates/matthews/partials/enum.tmpl.partial create mode 100644 eng/common/docgeneration/templates/matthews/partials/head.tmpl.partial create mode 100644 eng/common/docgeneration/templates/matthews/partials/namespace.tmpl.partial create mode 100644 eng/common/docgeneration/templates/matthews/styles/main.css create mode 100644 eng/common/docgeneration/templates/matthews/styles/main.js create mode 100644 eng/common/pipelines/codeowners-linter.yml create mode 100644 eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml create mode 100644 eng/common/pipelines/templates/jobs/docindex.yml create mode 100644 eng/common/pipelines/templates/jobs/generate-job-matrix.yml create mode 100644 eng/common/pipelines/templates/jobs/perf.yml create mode 100644 eng/common/pipelines/templates/jobs/prepare-pipelines.yml create mode 100644 eng/common/pipelines/templates/stages/archetype-sdk-tool-pwsh.yml create mode 100644 eng/common/pipelines/templates/steps/bypass-local-dns.yml create mode 100644 eng/common/pipelines/templates/steps/cache-ps-modules.yml create mode 100644 eng/common/pipelines/templates/steps/check-spelling.yml create mode 100644 eng/common/pipelines/templates/steps/cosmos-emulator.yml create mode 100644 eng/common/pipelines/templates/steps/create-apireview.yml create mode 100644 eng/common/pipelines/templates/steps/create-pull-request.yml create mode 100644 eng/common/pipelines/templates/steps/create-tags-and-git-release.yml create mode 100644 eng/common/pipelines/templates/steps/credscan.yml create mode 100644 eng/common/pipelines/templates/steps/daily-dev-build-variable.yml create mode 100644 eng/common/pipelines/templates/steps/detect-api-changes.yml create mode 100644 eng/common/pipelines/templates/steps/devops-variables-clear.yml create mode 100644 eng/common/pipelines/templates/steps/devops-variables-set.yml create mode 100644 eng/common/pipelines/templates/steps/docker-pull-image.yml create mode 100644 eng/common/pipelines/templates/steps/docsms-ensure-validation.yml create mode 100644 eng/common/pipelines/templates/steps/enable-long-path-support.yml create mode 100644 eng/common/pipelines/templates/steps/eng-common-workflow-enforcer.yml create mode 100644 eng/common/pipelines/templates/steps/git-push-changes.yml create mode 100644 eng/common/pipelines/templates/steps/install-pipeline-generation.yml create mode 100644 eng/common/pipelines/templates/steps/mashup-doc-index.yml create mode 100644 eng/common/pipelines/templates/steps/publish-1es-artifact.yml create mode 100644 eng/common/pipelines/templates/steps/publish-artifact.yml create mode 100644 eng/common/pipelines/templates/steps/publish-blobs.yml create mode 100644 eng/common/pipelines/templates/steps/replace-relative-links.yml create mode 100644 eng/common/pipelines/templates/steps/retain-run.yml create mode 100644 eng/common/pipelines/templates/steps/run-pester-tests.yml create mode 100644 eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml create mode 100644 eng/common/pipelines/templates/steps/set-default-branch.yml create mode 100644 eng/common/pipelines/templates/steps/set-test-pipeline-version.yml create mode 100644 eng/common/pipelines/templates/steps/sparse-checkout.yml create mode 100644 eng/common/pipelines/templates/steps/update-docsms-metadata.yml create mode 100644 eng/common/pipelines/templates/steps/validate-all-packages.yml create mode 100644 eng/common/pipelines/templates/steps/validate-filename.yml create mode 100644 eng/common/pipelines/templates/steps/verify-agent-os.yml create mode 100644 eng/common/pipelines/templates/steps/verify-changelog.yml create mode 100644 eng/common/pipelines/templates/steps/verify-links.yml create mode 100644 eng/common/pipelines/templates/steps/verify-path-length.yml create mode 100644 eng/common/pipelines/templates/steps/verify-readme.yml create mode 100644 eng/common/pipelines/templates/steps/verify-restapi-spec-location.yml create mode 100644 eng/common/pipelines/templates/steps/verify-samples.yml create mode 100644 eng/common/pipelines/templates/steps/write-filesystemmetrics.yml create mode 100644 eng/common/scripts/Add-IssueComment.ps1 create mode 100644 eng/common/scripts/Add-IssueLabels.ps1 create mode 100644 eng/common/scripts/Add-RetentionLease.ps1 create mode 100644 eng/common/scripts/Cadl-Project-Generate.ps1 create mode 100644 eng/common/scripts/Cadl-Project-Sync.ps1 create mode 100644 eng/common/scripts/ChangeLog-Operations.ps1 create mode 100644 eng/common/scripts/Cosmos-Emulator.ps1 create mode 100644 eng/common/scripts/Create-APIReview.ps1 create mode 100644 eng/common/scripts/Delete-RemoteBranches.ps1 create mode 100644 eng/common/scripts/Delete-RemoteTag.ps1 create mode 100644 eng/common/scripts/Detect-Api-Changes.ps1 create mode 100644 eng/common/scripts/Generate-PR-Diff.ps1 create mode 100644 eng/common/scripts/Get-BuildSourceDescription.ps1 create mode 100644 eng/common/scripts/Get-PullRequestCreator.ps1 create mode 100644 eng/common/scripts/Helpers/ApiView-Helpers.ps1 create mode 100644 eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1 create mode 100644 eng/common/scripts/Helpers/DevOps-WorkItem-Helpers.ps1 create mode 100644 eng/common/scripts/Helpers/Metadata-Helpers.ps1 create mode 100644 eng/common/scripts/Helpers/PSModule-Helpers.ps1 create mode 100644 eng/common/scripts/Helpers/Package-Helpers.ps1 create mode 100644 eng/common/scripts/Helpers/Resource-Helpers.ps1 create mode 100644 eng/common/scripts/Helpers/Service-Level-Readme-Automation-Helpers.ps1 create mode 100644 eng/common/scripts/Helpers/git-helpers.ps1 create mode 100644 eng/common/scripts/Helpers/git-helpers.tests.ps1 create mode 100644 eng/common/scripts/Import-AzModules.ps1 create mode 100644 eng/common/scripts/Invoke-DevOpsAPI.ps1 create mode 100644 eng/common/scripts/Invoke-GitHubAPI.ps1 create mode 100644 eng/common/scripts/New-RegenerateMatrix.ps1 create mode 100644 eng/common/scripts/New-ReleaseAsset.ps1 create mode 100644 eng/common/scripts/Package-Properties.ps1 create mode 100644 eng/common/scripts/Prepare-Release.ps1 create mode 100644 eng/common/scripts/Queue-Pipeline.ps1 create mode 100644 eng/common/scripts/Save-Package-Properties.ps1 create mode 100644 eng/common/scripts/SemVer.ps1 create mode 100644 eng/common/scripts/Service-Level-Readme-Automation.ps1 create mode 100644 eng/common/scripts/SetTestPipelineVersion.ps1 create mode 100644 eng/common/scripts/Submit-PullRequest.ps1 create mode 100644 eng/common/scripts/Test-SampleMetadata.ps1 create mode 100644 eng/common/scripts/TypeSpec-Project-Generate.ps1 create mode 100644 eng/common/scripts/TypeSpec-Project-Process.ps1 create mode 100644 eng/common/scripts/TypeSpec-Project-Sync.ps1 create mode 100644 eng/common/scripts/Update-ChangeLog.ps1 create mode 100644 eng/common/scripts/Update-DevOps-Release-WorkItem.ps1 create mode 100644 eng/common/scripts/Update-DocsMsMetadata.ps1 create mode 100644 eng/common/scripts/Update-DocsMsPackageMonikers.ps1 create mode 100644 eng/common/scripts/Update-DocsMsPackages.ps1 create mode 100644 eng/common/scripts/Update-DocsMsToc.ps1 create mode 100644 eng/common/scripts/Update-GeneratedSdks.ps1 create mode 100644 eng/common/scripts/Validate-All-Packages.ps1 create mode 100644 eng/common/scripts/Validate-Package.ps1 create mode 100644 eng/common/scripts/Verify-AgentOS.ps1 create mode 100644 eng/common/scripts/Verify-ChangeLog.ps1 create mode 100644 eng/common/scripts/Verify-Links.ps1 create mode 100644 eng/common/scripts/Verify-Readme.ps1 create mode 100644 eng/common/scripts/Verify-RequiredDocsJsonMembers.ps1 create mode 100644 eng/common/scripts/Verify-Resource-Ref.ps1 create mode 100644 eng/common/scripts/Verify-RestApiSpecLocation.ps1 create mode 100644 eng/common/scripts/Write-FileSystemMetrics.ps1 create mode 100644 eng/common/scripts/X509Certificate2/README.md create mode 100644 eng/common/scripts/X509Certificate2/X509Certificate2.psm1 create mode 100644 eng/common/scripts/artifact-metadata-parsing.ps1 create mode 100644 eng/common/scripts/check-for-git-changes.ps1 create mode 100644 eng/common/scripts/check-spelling-in-changed-files.ps1 create mode 100644 eng/common/scripts/common.ps1 create mode 100644 eng/common/scripts/copy-docs-to-blobstorage.ps1 create mode 100644 eng/common/scripts/create-tags-and-git-release.ps1 create mode 100644 eng/common/scripts/get-changedfiles.ps1 create mode 100644 eng/common/scripts/get-markdown-files-from-changed-files.ps1 create mode 100644 eng/common/scripts/git-branch-push.ps1 create mode 100644 eng/common/scripts/job-matrix/Create-JobMatrix.ps1 create mode 100644 eng/common/scripts/job-matrix/README.md create mode 100644 eng/common/scripts/job-matrix/job-matrix-functions.ps1 create mode 100644 eng/common/scripts/logging.ps1 create mode 100644 eng/common/scripts/stress-testing/deploy-stress-tests.ps1 create mode 100644 eng/common/scripts/stress-testing/find-all-stress-packages.ps1 create mode 100644 eng/common/scripts/stress-testing/generate-scenario-matrix.ps1 create mode 100644 eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 create mode 100644 eng/common/scripts/trust-proxy-certificate.ps1 create mode 100644 eng/common/scripts/typespec/New-EmitterPackageJson.ps1 create mode 100644 eng/common/scripts/typespec/New-EmitterPackageLock.ps1 create mode 100644 eng/common/spelling/Invoke-Cspell.ps1 create mode 100644 eng/common/spelling/package-lock.json create mode 100644 eng/common/spelling/package.json create mode 100644 eng/common/testproxy/apply-dev-cert.sh create mode 100644 eng/common/testproxy/docker-start-proxy.ps1 create mode 100644 eng/common/testproxy/dotnet-devcert.crt create mode 100644 eng/common/testproxy/dotnet-devcert.pfx create mode 100644 eng/common/testproxy/localhost.conf create mode 100644 eng/common/testproxy/onboarding/README.md create mode 100644 eng/common/testproxy/onboarding/common-asset-functions.ps1 create mode 100644 eng/common/testproxy/onboarding/generate-assets-json.ps1 create mode 100644 eng/common/testproxy/publish-proxy-logs.yml create mode 100644 eng/common/testproxy/scripts/override-proxy-version.ps1 create mode 100644 eng/common/testproxy/scripts/resolve-asset-conflict/README.md create mode 100644 eng/common/testproxy/scripts/resolve-asset-conflict/resolve-asset-conflict.ps1 create mode 100644 eng/common/testproxy/scripts/tag-merge/README.md create mode 100644 eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1 create mode 100644 eng/common/testproxy/target_version.txt create mode 100644 eng/common/testproxy/test-proxy-docker.yml create mode 100644 eng/common/testproxy/test-proxy-tool-shutdown.yml create mode 100644 eng/common/testproxy/test-proxy-tool.yml diff --git a/eng/common/InterdependencyGraph.html b/eng/common/InterdependencyGraph.html new file mode 100644 index 00000000..21f78563 --- /dev/null +++ b/eng/common/InterdependencyGraph.html @@ -0,0 +1,356 @@ + + + +Interdependency Graph + + + + + + + + +
+

Dependency Graph

+ + +
+
+
+ + + diff --git a/eng/common/README.md b/eng/common/README.md new file mode 100644 index 00000000..732688da --- /dev/null +++ b/eng/common/README.md @@ -0,0 +1,3 @@ +# Common Engineering System + +Updates under this directory should only be made in the `azure-sdk-tools` repo as any changes under this directory outside of that repo will end up getting overwritten with future updates. For information about making updates see [common engineering system docs](https://github.com/Azure/azure-sdk-tools/blob/main/doc/common/common_engsys.md) diff --git a/eng/common/TestResources/New-TestResources.cmd b/eng/common/TestResources/New-TestResources.cmd new file mode 100644 index 00000000..94b0c1f2 --- /dev/null +++ b/eng/common/TestResources/New-TestResources.cmd @@ -0,0 +1,17 @@ +@echo off + +REM Copyright (c) Microsoft Corporation. All rights reserved. +REM Licensed under the MIT License. + +setlocal + +for /f "usebackq delims=" %%i in (`where pwsh 2^>nul`) do ( + set _cmd=%%i +) + +if "%_cmd%"=="" ( + echo Error: PowerShell not found. Please visit https://github.com/powershell/powershell for install instructions. + exit /b 2 +) + +call "%_cmd%" -NoLogo -NoProfile -File "%~dpn0.ps1" %* diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1 new file mode 100644 index 00000000..ae38aefa --- /dev/null +++ b/eng/common/TestResources/New-TestResources.ps1 @@ -0,0 +1,1122 @@ +#!/usr/bin/env pwsh + +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +#Requires -Version 6.0 +#Requires -PSEdition Core +#Requires -Modules @{ModuleName='Az.Accounts'; ModuleVersion='1.6.4'} +#Requires -Modules @{ModuleName='Az.Resources'; ModuleVersion='1.8.0'} + +[CmdletBinding(DefaultParameterSetName = 'Default', SupportsShouldProcess = $true, ConfirmImpact = 'Medium')] +param ( + # Limit $BaseName to enough characters to be under limit plus prefixes, and https://docs.microsoft.com/azure/architecture/best-practices/resource-naming + [Parameter()] + [ValidatePattern('^[-a-zA-Z0-9\.\(\)_]{0,80}(?<=[a-zA-Z0-9\(\)])$')] + [string] $BaseName, + + [ValidatePattern('^[-\w\._\(\)]+$')] + [string] $ResourceGroupName, + + [Parameter(Mandatory = $true, Position = 0)] + [string] $ServiceDirectory, + + [Parameter()] + [string] $TestResourcesDirectory, + + [Parameter()] + [ValidatePattern('^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$')] + [string] $TestApplicationId, + + [Parameter()] + [string] $TestApplicationSecret, + + [Parameter()] + [ValidatePattern('^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$')] + [string] $TestApplicationOid, + + [Parameter(ParameterSetName = 'Provisioner', Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] $TenantId, + + # Azure SDK Developer Playground subscription is assumed if not set + [Parameter()] + [ValidatePattern('^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$')] + [string] $SubscriptionId, + + [Parameter(ParameterSetName = 'Provisioner', Mandatory = $true)] + [ValidatePattern('^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$')] + [string] $ProvisionerApplicationId, + + [Parameter(ParameterSetName = 'Provisioner', Mandatory = $false)] + [ValidatePattern('^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$')] + [string] $ProvisionerApplicationOid, + + [Parameter(ParameterSetName = 'Provisioner', Mandatory = $true)] + [string] $ProvisionerApplicationSecret, + + [Parameter()] + [ValidateRange(1, 7*24)] + [int] $DeleteAfterHours = 120, + + [Parameter()] + [string] $Location = '', + + [Parameter()] + [ValidateSet('AzureCloud', 'AzureUSGovernment', 'AzureChinaCloud', 'Dogfood')] + [string] $Environment = 'AzureCloud', + + [Parameter()] + [ValidateSet('test', 'perf', 'stress-test')] + [string] $ResourceType = 'test', + + [Parameter()] + [hashtable] $ArmTemplateParameters, + + [Parameter()] + [hashtable] $AdditionalParameters, + + [Parameter()] + [ValidateNotNull()] + [hashtable] $EnvironmentVariables = @{}, + + [Parameter()] + [switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID), + + [Parameter()] + [switch] $Force, + + [Parameter()] + [switch] $OutFile, + + [Parameter()] + [switch] $SuppressVsoCommands = ($null -eq $env:SYSTEM_TEAMPROJECTID), + + [Parameter()] + [switch] $UserAuth, + + # Captures any arguments not declared here (no parameter errors) + # This enables backwards compatibility with old script versions in + # hotfix branches if and when the dynamic subscription configuration + # secrets get updated to add new parameters. + [Parameter(ValueFromRemainingArguments = $true)] + $NewTestResourcesRemainingArguments +) + +. $PSScriptRoot/SubConfig-Helpers.ps1 + +# By default stop for any error. +if (!$PSBoundParameters.ContainsKey('ErrorAction')) { + $ErrorActionPreference = 'Stop' +} + +function Log($Message) +{ + Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message) +} + +# vso commands are specially formatted log lines that are parsed by Azure Pipelines +# to perform additional actions, most commonly marking values as secrets. +# https://docs.microsoft.com/en-us/azure/devops/pipelines/scripts/logging-commands +function LogVsoCommand([string]$message) +{ + if (!$CI -or $SuppressVsoCommands) { + return + } + Write-Host $message +} + +function Retry([scriptblock] $Action, [int] $Attempts = 5) +{ + $attempt = 0 + $sleep = 5 + + while ($attempt -lt $Attempts) { + try { + $attempt++ + return $Action.Invoke() + } catch { + if ($attempt -lt $Attempts) { + $sleep *= 2 + + Write-Warning "Attempt $attempt failed: $_. Trying again in $sleep seconds..." + Start-Sleep -Seconds $sleep + } else { + throw + } + } + } +} + +# NewServicePrincipalWrapper creates an object from an AAD graph or Microsoft Graph service principal object type. +# This is necessary to work around breaking changes introduced in Az version 7.0.0: +# https://azure.microsoft.com/en-us/updates/update-your-apps-to-use-microsoft-graph-before-30-june-2022/ +function NewServicePrincipalWrapper([string]$subscription, [string]$resourceGroup, [string]$displayName) +{ + if ((Get-Module Az.Resources).Version -eq "5.3.0") { + # https://github.com/Azure/azure-powershell/issues/17040 + # New-AzAdServicePrincipal calls will fail with: + # "You cannot call a method on a null-valued expression." + Write-Warning "Az.Resources version 5.3.0 is not supported. Please update to >= 5.3.1" + Write-Warning "Update-Module Az.Resources -RequiredVersion 5.3.1" + exit 1 + } + + try { + $servicePrincipal = Retry { + New-AzADServicePrincipal -Role "Owner" -Scope "/subscriptions/$SubscriptionId/resourceGroups/$ResourceGroupName" -DisplayName $displayName + } + } catch { + # The underlying error "The directory object quota limit for the Principal has been exceeded" gets overwritten by the module trying + # to call New-AzADApplication with a null object instead of stopping execution, which makes this case hard to diagnose because it prints the following: + # "Cannot bind argument to parameter 'ObjectId' because it is an empty string." + # Provide a more helpful diagnostic prompt to the user if appropriate: + $totalApps = (Get-AzADApplication -OwnedApplication).Length + $msg = "App Registrations owned by you total $totalApps and may exceed the max quota (likely around 135)." + ` + "`nTry removing some at https://ms.portal.azure.com/#view/Microsoft_AAD_IAM/ActiveDirectoryMenuBlade/~/RegisteredApps" + ` + " or by running the following command to remove apps created by this script:" + ` + "`n Get-AzADApplication -DisplayNameStartsWith '$baseName' | Remove-AzADApplication" + ` + "`nNOTE: You may need to wait for the quota number to be updated after removing unused applications." + Write-Warning $msg + throw + } + + $spPassword = "" + $appId = "" + if (Get-Member -Name "Secret" -InputObject $servicePrincipal -MemberType property) { + Write-Verbose "Using legacy PSADServicePrincipal object type from AAD graph API" + # Secret property exists on PSADServicePrincipal type from AAD graph in Az # module versions < 7.0.0 + $spPassword = $servicePrincipal.Secret + $appId = $servicePrincipal.ApplicationId + } else { + if ((Get-Module Az.Resources).Version -eq "5.1.0") { + Write-Verbose "Creating password and credential for service principal via MS Graph API" + Write-Warning "Please update Az.Resources to >= 5.2.0 by running 'Update-Module Az'" + # Microsoft graph objects (Az.Resources version == 5.1.0) do not provision a secret on creation so it must be added separately. + # Submitting a password credential object without specifying a password will result in one being generated on the server side. + $password = New-Object -TypeName "Microsoft.Azure.PowerShell.Cmdlets.Resources.MSGraph.Models.ApiV10.MicrosoftGraphPasswordCredential" + $password.DisplayName = "Password for $displayName" + $credential = Retry { New-AzADSpCredential -PasswordCredentials $password -ServicePrincipalObject $servicePrincipal -ErrorAction 'Stop' } + $spPassword = ConvertTo-SecureString $credential.SecretText -AsPlainText -Force + $appId = $servicePrincipal.AppId + } else { + Write-Verbose "Creating service principal credential via MS Graph API" + # In 5.2.0 the password credential issue was fixed (see https://github.com/Azure/azure-powershell/pull/16690) but the + # parameter set was changed making the above call fail due to a missing ServicePrincipalId parameter. + $credential = Retry { $servicePrincipal | New-AzADSpCredential -ErrorAction 'Stop' } + $spPassword = ConvertTo-SecureString $credential.SecretText -AsPlainText -Force + $appId = $servicePrincipal.AppId + } + } + + return @{ + AppId = $appId + ApplicationId = $appId + # This is the ObjectId/OID but most return objects use .Id so keep it consistent to prevent confusion + Id = $servicePrincipal.Id + DisplayName = $servicePrincipal.DisplayName + Secret = $spPassword + } +} + +function LoadCloudConfig([string] $env) +{ + $configPath = "$PSScriptRoot/clouds/$env.json" + if (!(Test-Path $configPath)) { + Write-Warning "Could not find cloud configuration for environment '$env'" + return @{} + } + + $config = Get-Content $configPath | ConvertFrom-Json -AsHashtable + return $config +} + +function MergeHashes([hashtable] $source, [psvariable] $dest) +{ + foreach ($key in $source.Keys) { + if ($dest.Value.Contains($key) -and $dest.Value[$key] -ne $source[$key]) { + Write-Warning ("Overwriting '$($dest.Name).$($key)' with value '$($dest.Value[$key])' " + + "to new value '$($source[$key])'") + } + $dest.Value[$key] = $source[$key] + } +} + +function BuildBicepFile([System.IO.FileSystemInfo] $file) +{ + if (!(Get-Command bicep -ErrorAction Ignore)) { + Write-Error "A bicep file was found at '$($file.FullName)' but the Azure Bicep CLI is not installed. See https://aka.ms/install-bicep-pwsh" + throw + } + + $tmp = $env:TEMP ? $env:TEMP : [System.IO.Path]::GetTempPath() + $templateFilePath = Join-Path $tmp "$ResourceType-resources.$(New-Guid).compiled.json" + + # Az can deploy bicep files natively, but by compiling here it becomes easier to parse the + # outputted json for mismatched parameter declarations. + bicep build $file.FullName --outfile $templateFilePath + if ($LASTEXITCODE) { + Write-Error "Failure building bicep file '$($file.FullName)'" + throw + } + + return $templateFilePath +} + +function BuildDeploymentOutputs([string]$serviceName, [object]$azContext, [object]$deployment, [hashtable]$environmentVariables) { + $serviceDirectoryPrefix = BuildServiceDirectoryPrefix $serviceName + # Add default values + $deploymentOutputs = [Ordered]@{ + "${serviceDirectoryPrefix}CLIENT_ID" = $TestApplicationId; + "${serviceDirectoryPrefix}CLIENT_SECRET" = $TestApplicationSecret; + "${serviceDirectoryPrefix}TENANT_ID" = $azContext.Tenant.Id; + "${serviceDirectoryPrefix}SUBSCRIPTION_ID" = $azContext.Subscription.Id; + "${serviceDirectoryPrefix}RESOURCE_GROUP" = $resourceGroup.ResourceGroupName; + "${serviceDirectoryPrefix}LOCATION" = $resourceGroup.Location; + "${serviceDirectoryPrefix}ENVIRONMENT" = $azContext.Environment.Name; + "${serviceDirectoryPrefix}AZURE_AUTHORITY_HOST" = $azContext.Environment.ActiveDirectoryAuthority; + "${serviceDirectoryPrefix}RESOURCE_MANAGER_URL" = $azContext.Environment.ResourceManagerUrl; + "${serviceDirectoryPrefix}SERVICE_MANAGEMENT_URL" = $azContext.Environment.ServiceManagementUrl; + "AZURE_SERVICE_DIRECTORY" = $serviceName.ToUpperInvariant(); + } + + MergeHashes $environmentVariables $(Get-Variable deploymentOutputs) + + foreach ($key in $deployment.Outputs.Keys) { + $variable = $deployment.Outputs[$key] + + # Work around bug that makes the first few characters of environment variables be lowercase. + $key = $key.ToUpperInvariant() + + if ($variable.Type -eq 'String' -or $variable.Type -eq 'SecureString') { + $deploymentOutputs[$key] = $variable.Value + } + } + + # Force capitalization of all keys to avoid Azure Pipelines confusion with + # variable auto-capitalization and OS env var capitalization differences + $capitalized = @{} + foreach ($item in $deploymentOutputs.GetEnumerator()) { + $capitalized[$item.Name.ToUpperInvariant()] = $item.Value + } + + return $capitalized +} + +function SetDeploymentOutputs( + [string]$serviceName, + [object]$azContext, + [object]$deployment, + [object]$templateFile, + [hashtable]$environmentVariables = @{} +) { + $deploymentEnvironmentVariables = $environmentVariables.Clone() + $deploymentOutputs = BuildDeploymentOutputs $serviceName $azContext $deployment $deploymentEnvironmentVariables + + if ($OutFile) { + if (!$IsWindows) { + Write-Host 'File option is supported only on Windows' + } + + $outputFile = "$($templateFile.originalFilePath).env" + + $environmentText = $deploymentOutputs | ConvertTo-Json; + $bytes = [System.Text.Encoding]::UTF8.GetBytes($environmentText) + $protectedBytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser) + + Set-Content $outputFile -Value $protectedBytes -AsByteStream -Force + + Write-Host "Test environment settings`n $environmentText`nstored into encrypted $outputFile" + } else { + if (!$CI) { + # Write an extra new line to isolate the environment variables for easy reading. + Log "Persist the following environment variables based on your detected shell ($shell):`n" + } + + # Write overwrite warnings first, since local execution prints a runnable command to export variables + foreach ($key in $deploymentOutputs.Keys) { + if ([Environment]::GetEnvironmentVariable($key)) { + Write-Warning "Deployment outputs will overwrite pre-existing environment variable '$key'" + } + } + + # Marking values as secret by allowed keys below is not sufficient, as there may be outputs set in the ARM/bicep + # file that re-mark those values as secret (since all user-provided deployment outputs are treated as secret by default). + # This variable supports a second check on not marking previously allowed keys/values as secret. + $notSecretValues = @() + foreach ($key in $deploymentOutputs.Keys) { + $value = $deploymentOutputs[$key] + $deploymentEnvironmentVariables[$key] = $value + + if ($CI) { + if (ShouldMarkValueAsSecret $serviceName $key $value $notSecretValues) { + # Treat all ARM template output variables as secrets since "SecureString" variables do not set values. + # In order to mask secrets but set environment variables for any given ARM template, we set variables twice as shown below. + LogVsoCommand "##vso[task.setvariable variable=_$key;issecret=true;]$value" + Write-Host "Setting variable as secret '$key'" + } else { + Write-Host "Setting variable '$key': $value" + $notSecretValues += $value + } + LogVsoCommand "##vso[task.setvariable variable=$key;]$value" + } else { + Write-Host ($shellExportFormat -f $key, $value) + } + } + + if ($key) { + # Isolate the environment variables for easy reading. + Write-Host "`n" + $key = $null + } + } + + return $deploymentEnvironmentVariables, $deploymentOutputs +} + +# Support actions to invoke on exit. +$exitActions = @({ + if ($exitActions.Count -gt 1) { + Write-Verbose 'Running registered exit actions' + } +}) + +New-Variable -Name 'initialContext' -Value (Get-AzContext) -Option Constant +if ($initialContext) { + $exitActions += { + Write-Verbose "Restoring initial context: $($initialContext.Account)" + $null = $initialContext | Select-AzContext + } +} + +# try..finally will also trap Ctrl+C. +try { + + # Enumerate test resources to deploy. Fail if none found. + $repositoryRoot = "$PSScriptRoot/../../.." | Resolve-Path + $root = [System.IO.Path]::Combine($repositoryRoot, "sdk", $ServiceDirectory) | Resolve-Path + if ($TestResourcesDirectory) { + $root = $TestResourcesDirectory | Resolve-Path + # Add an explicit check below in case ErrorActionPreference is overridden and Resolve-Path doesn't stop execution + if (!$root) { + throw "TestResourcesDirectory '$TestResourcesDirectory' does not exist." + } + Write-Verbose "Overriding test resources search directory to '$root'" + } + $templateFiles = @() + + "$ResourceType-resources.json", "$ResourceType-resources.bicep" | ForEach-Object { + Write-Verbose "Checking for '$_' files under '$root'" + Get-ChildItem -Path $root -Filter "$_" -Recurse | ForEach-Object { + Write-Verbose "Found template '$($_.FullName)'" + if ($_.Extension -eq '.bicep') { + $templateFile = @{originalFilePath = $_.FullName; jsonFilePath = (BuildBicepFile $_)} + $templateFiles += $templateFile + } else { + $templateFile = @{originalFilePath = $_.FullName; jsonFilePath = $_.FullName} + $templateFiles += $templateFile + } + } + } + + if (!$templateFiles) { + Write-Warning -Message "No template files found under '$root'" + exit + } + + $serviceName = GetServiceLeafDirectoryName $ServiceDirectory + $BaseName, $ResourceGroupName = GetBaseAndResourceGroupNames ` + -baseNameDefault $BaseName ` + -resourceGroupNameDefault $ResourceGroupName ` + -user (GetUserName) ` + -serviceDirectoryName $serviceName ` + -CI $CI + + # Make sure pre- and post-scripts are passed formerly required arguments. + $PSBoundParameters['BaseName'] = $BaseName + + # Try detecting repos that support OutFile and defaulting to it + if (!$CI -and !$PSBoundParameters.ContainsKey('OutFile') -and $IsWindows) { + # TODO: find a better way to detect the language + if (Test-Path "$repositoryRoot/eng/service.proj") { + $OutFile = $true + Log "Detected .NET repository. Defaulting OutFile to true. Test environment settings would be stored into the file so you don't need to set environment variables manually." + } + } + + # If no location is specified use safe default locations for the given + # environment. If no matching environment is found $Location remains an empty + # string. + if (!$Location) { + $Location = @{ + 'AzureCloud' = 'westus'; + 'AzureUSGovernment' = 'usgovvirginia'; + 'AzureChinaCloud' = 'chinaeast2'; + 'Dogfood' = 'westus' + }[$Environment] + + Write-Verbose "Location was not set. Using default location for environment: '$Location'" + } + + if (!$CI -and $PSCmdlet.ParameterSetName -ne "Provisioner") { + # Make sure the user is logged in to create a service principal. + $context = Get-AzContext; + if (!$context) { + Log 'User not logged in. Logging in now...' + $context = (Connect-AzAccount).Context + } + + $currentSubcriptionId = $context.Subscription.Id + + # If no subscription was specified, try to select the Azure SDK Developer Playground subscription. + # Ignore errors to leave the automatically selected subscription. + if ($SubscriptionId) { + if ($currentSubcriptionId -ne $SubscriptionId) { + Log "Selecting subscription '$SubscriptionId'" + $null = Select-AzSubscription -Subscription $SubscriptionId + + $exitActions += { + Log "Selecting previous subscription '$currentSubcriptionId'" + $null = Select-AzSubscription -Subscription $currentSubcriptionId + } + + # Update the context. + $context = Get-AzContext + } + } else { + if ($currentSubcriptionId -ne 'faa080af-c1d8-40ad-9cce-e1a450ca5b57') { + Log "Attempting to select subscription 'Azure SDK Developer Playground (faa080af-c1d8-40ad-9cce-e1a450ca5b57)'" + $null = Select-AzSubscription -Subscription 'faa080af-c1d8-40ad-9cce-e1a450ca5b57' -ErrorAction Ignore + + # Update the context. + $context = Get-AzContext + } + + $SubscriptionId = $context.Subscription.Id + $PSBoundParameters['SubscriptionId'] = $SubscriptionId + } + + # Use cache of well-known team subs without having to be authenticated. + $wellKnownSubscriptions = @{ + 'faa080af-c1d8-40ad-9cce-e1a450ca5b57' = 'Azure SDK Developer Playground' + 'a18897a6-7e44-457d-9260-f2854c0aca42' = 'Azure SDK Engineering System' + '2cd617ea-1866-46b1-90e3-fffb087ebf9b' = 'Azure SDK Test Resources' + } + + # Print which subscription is currently selected. + $subscriptionName = $context.Subscription.Id + if ($wellKnownSubscriptions.ContainsKey($subscriptionName)) { + $subscriptionName = '{0} ({1})' -f $wellKnownSubscriptions[$subscriptionName], $subscriptionName + } + + Log "Using subscription '$subscriptionName'" + + # Make sure the TenantId is also updated from the current context. + # PSBoundParameters is not updated to avoid confusing parameter sets. + if (!$TenantId) { + $TenantId = $context.Subscription.TenantId + } + } + + # If a provisioner service principal was provided, log into it to perform the pre- and post-scripts and deployments. + if ($ProvisionerApplicationId) { + $null = Disable-AzContextAutosave -Scope Process + + Log "Logging into service principal '$ProvisionerApplicationId'." + $provisionerSecret = ConvertTo-SecureString -String $ProvisionerApplicationSecret -AsPlainText -Force + $provisionerCredential = [System.Management.Automation.PSCredential]::new($ProvisionerApplicationId, $provisionerSecret) + + # Use the given subscription ID if provided. + $subscriptionArgs = if ($SubscriptionId) { + @{Subscription = $SubscriptionId} + } else { + @{} + } + + $provisionerAccount = Retry { + Connect-AzAccount -Force:$Force -Tenant $TenantId -Credential $provisionerCredential -ServicePrincipal -Environment $Environment @subscriptionArgs + } + + $exitActions += { + Write-Verbose "Logging out of service principal '$($provisionerAccount.Context.Account)'" + + # Only attempt to disconnect if the -WhatIf flag was not set. Otherwise, this call is not necessary and will fail. + if ($PSCmdlet.ShouldProcess($ProvisionerApplicationId)) { + $null = Disconnect-AzAccount -AzureContext $provisionerAccount.Context + } + } + } + + # Determine the Azure context that the script is running in. + $context = Get-AzContext; + + # Make sure the provisioner OID is set so we can pass it through to the deployment. + if (!$ProvisionerApplicationId -and !$ProvisionerApplicationOid) { + if ($context.Account.Type -eq 'User') { + $user = Get-AzADUser -UserPrincipalName $context.Account.Id + $ProvisionerApplicationOid = $user.Id + } elseif ($context.Account.Type -eq 'ServicePrincipal') { + $sp = Get-AzADServicePrincipal -ApplicationId $context.Account.Id + $ProvisionerApplicationOid = $sp.Id + } else { + Write-Warning "Getting the OID for provisioner type '$($context.Account.Type)' is not supported and will not be passed to deployments (seldom required)." + } + } elseif (!$ProvisionerApplicationOid) { + $sp = Get-AzADServicePrincipal -ApplicationId $ProvisionerApplicationId + $ProvisionerApplicationOid = $sp.Id + } + + $tags = @{ + Owners = (GetUserName) + ServiceDirectory = $ServiceDirectory + } + + # Tag the resource group to be deleted after a certain number of hours. + Write-Warning "Any clean-up scripts running against subscription '$SubscriptionId' may delete resource group '$ResourceGroupName' after $DeleteAfterHours hours." + $deleteAfter = [DateTime]::UtcNow.AddHours($DeleteAfterHours).ToString('o') + $tags['DeleteAfter'] = $deleteAfter + + if ($CI) { + # Add tags for the current CI job. + $tags += @{ + BuildId = "${env:BUILD_BUILDID}" + BuildJob = "${env:AGENT_JOBNAME}" + BuildNumber = "${env:BUILD_BUILDNUMBER}" + BuildReason = "${env:BUILD_REASON}" + } + + # Set an environment variable marking that resources have been deployed + # This variable can be consumed as a yaml condition in later stages of the pipeline + # to determine whether resources should be removed. + Write-Host "Setting variable 'CI_HAS_DEPLOYED_RESOURCES': 'true'" + LogVsoCommand "##vso[task.setvariable variable=CI_HAS_DEPLOYED_RESOURCES;]true" + } + + Log "Creating resource group '$ResourceGroupName' in location '$Location'" + $resourceGroup = Retry { + New-AzResourceGroup -Name "$ResourceGroupName" -Location $Location -Tag $tags -Force:$Force + } + + if ($resourceGroup.ProvisioningState -eq 'Succeeded') { + # New-AzResourceGroup would've written an error and stopped the pipeline by default anyway. + Write-Verbose "Successfully created resource group '$($resourceGroup.ResourceGroupName)'" + } elseif (!$resourceGroup) { + if (!$PSCmdlet.ShouldProcess($resourceGroupName)) { + # If the -WhatIf flag was passed, there will be no resource group created. Fake it. + $resourceGroup = [PSCustomObject]@{ + ResourceGroupName = $resourceGroupName + Location = $Location + } + } else { + Write-Error "Resource group '$ResourceGroupName' already exists." ` + -Category ResourceExists ` + -RecommendedAction "Delete resource group '$ResourceGroupName', or overwrite it when redeploying." + } + } + + if ($UserAuth) { + if ($TestApplicationId) { + Write-Warning "The specified TestApplicationId '$TestApplicationId' will be ignored when UserAuth is set." + } + + $userAccount = (Get-AzADUser -UserPrincipalName (Get-AzContext).Account) + $TestApplicationOid = $userAccount.Id + $TestApplicationId = $testApplicationOid + $userAccountName = $userAccount.UserPrincipalName + Log "User authentication with user '$userAccountName' ('$TestApplicationId') will be used." + } + # If no test application ID was specified during an interactive session, create a new service principal. + elseif (!$CI -and !$TestApplicationId) { + # Cache the created service principal in this session for frequent reuse. + $servicePrincipal = if ($AzureTestPrincipal -and (Get-AzADServicePrincipal -ApplicationId $AzureTestPrincipal.AppId) -and $AzureTestSubscription -eq $SubscriptionId) { + Log "TestApplicationId was not specified; loading cached service principal '$($AzureTestPrincipal.AppId)'" + $AzureTestPrincipal + } else { + Log "TestApplicationId was not specified; creating a new service principal in subscription '$SubscriptionId'" + $suffix = (New-Guid).ToString('n').Substring(0, 4) + + # Service principals in the Microsoft AAD tenant must end with an @microsoft.com domain; those in other tenants + # are not permitted to do so. Format the non-MS name so there's an assocation with the Azure SDK. + if ($TenantId -eq '72f988bf-86f1-41af-91ab-2d7cd011db47') { + $displayName = "$ResourceType-resources-$($baseName)$suffix.microsoft.com" + } else { + $displayName = "$($baseName)$suffix.$ResourceType-resources.azure.sdk" + } + + $servicePrincipalWrapper = NewServicePrincipalWrapper ` + -subscription $SubscriptionId ` + -resourceGroup $ResourceGroupName ` + -displayName $DisplayName + + $global:AzureTestPrincipal = $servicePrincipalWrapper + $global:AzureTestSubscription = $SubscriptionId + + Log "Created service principal. AppId: '$($AzureTestPrincipal.AppId)' ObjectId: '$($AzureTestPrincipal.Id)'" + $servicePrincipalWrapper + $resourceGroupRoleAssigned = $true + } + + $TestApplicationId = $servicePrincipal.AppId + $TestApplicationOid = $servicePrincipal.Id + $TestApplicationSecret = (ConvertFrom-SecureString $servicePrincipal.Secret -AsPlainText) + } + + # Get test application OID from ID if not already provided. This may fail if the + # provisioner is a service principal without permissions to query AAD. This is a + # critical failure, but we should prompt with possible remediation. + if ($TestApplicationId -and !$TestApplicationOid) { + Log "Attempting to query the Object ID for the test service principal" + + try { + $testServicePrincipal = Retry { + Get-AzADServicePrincipal -ApplicationId $TestApplicationId + } + } + catch { + Write-Warning ("The Object ID of the test application was unable to be queried. " + + "You may want to consider passing it explicitly with the 'TestApplicationOid` parameter.") + throw $_.Exception + } + + if ($testServicePrincipal -and $testServicePrincipal.Id) { + $script:TestApplicationOid = $testServicePrincipal.Id + } + } + + # Make sure pre- and post-scripts are passed formerly required arguments. + $PSBoundParameters['TestApplicationId'] = $TestApplicationId + $PSBoundParameters['TestApplicationOid'] = $TestApplicationOid + $PSBoundParameters['TestApplicationSecret'] = $TestApplicationSecret + + # If the role hasn't been explicitly assigned to the resource group and a cached service principal or user authentication is in use, + # query to see if the grant is needed. + if (!$resourceGroupRoleAssigned -and $TestApplicationOid) { + $roleAssignment = Get-AzRoleAssignment ` + -ObjectId $TestApplicationOid ` + -RoleDefinitionName 'Owner' ` + -ResourceGroupName "$ResourceGroupName" ` + -ErrorAction SilentlyContinue + $resourceGroupRoleAssigned = ($roleAssignment.RoleDefinitionName -eq 'Owner') + } + + # If needed, grant the test service principal ownership over the resource group. This may fail if the provisioner + # is a service principal without permissions to grant RBAC roles to other service principals. That should not be + # considered a critical failure, as the test application may have subscription-level permissions and not require + # the explicit grant. + if (!$resourceGroupRoleAssigned) { + $idSlug = if ($userAuth) { "User '$userAccountName' ('$TestApplicationId')"} else { "Test Application '$TestApplicationId'"}; + Log "Attempting to assign the 'Owner' role for '$ResourceGroupName' to the $idSlug" + $ownerAssignment = New-AzRoleAssignment ` + -RoleDefinitionName "Owner" ` + -ObjectId "$TestApplicationOId" ` + -ResourceGroupName "$ResourceGroupName" ` + -ErrorAction SilentlyContinue + + if ($ownerAssignment.RoleDefinitionName -eq 'Owner') { + Write-Verbose "Successfully assigned ownership of '$ResourceGroupName' to the $idSlug" + } else { + Write-Warning ("The 'Owner' role for '$ResourceGroupName' could not be assigned. " + + "You may need to manually grant 'Owner' for the resource group to the " + + "$idSlug if it does not have subscription-level permissions.") + } + } + + # Populate the template parameters and merge any additional specified. + $templateParameters = @{ + baseName = $BaseName + testApplicationId = $TestApplicationId + testApplicationOid = "$TestApplicationOid" + } + if ($ProvisionerApplicationOid) { + $templateParameters["provisionerApplicationOid"] = "$ProvisionerApplicationOid" + } + + if ($TenantId) { + $templateParameters.Add('tenantId', $TenantId) + } + if ($TestApplicationSecret) { + $templateParameters.Add('testApplicationSecret', $TestApplicationSecret) + } + + $defaultCloudParameters = LoadCloudConfig $Environment + MergeHashes $defaultCloudParameters $(Get-Variable templateParameters) + MergeHashes $ArmTemplateParameters $(Get-Variable templateParameters) + MergeHashes $AdditionalParameters $(Get-Variable templateParameters) + + # Include environment-specific parameters only if not already provided as part of the "ArmTemplateParameters" + if (($context.Environment.StorageEndpointSuffix) -and (-not ($templateParameters.ContainsKey('storageEndpointSuffix')))) { + $templateParameters.Add('storageEndpointSuffix', $context.Environment.StorageEndpointSuffix) + } + + # Try to detect the shell based on the parent process name (e.g. launch via shebang). + $shell, $shellExportFormat = if (($parentProcessName = (Get-Process -Id $PID).Parent.ProcessName) -and $parentProcessName -eq 'cmd') { + 'cmd', 'set {0}={1}' + } elseif (@('bash', 'csh', 'tcsh', 'zsh') -contains $parentProcessName) { + 'shell', 'export {0}={1}' + } else { + 'PowerShell', '${{env:{0}}} = ''{1}''' + } + + # Deploy the templates + foreach ($templateFile in $templateFiles) { + # Deployment fails if we pass in more parameters than are defined. + Write-Verbose "Removing unnecessary parameters from template '$($templateFile.jsonFilePath)'" + $templateJson = Get-Content -LiteralPath $templateFile.jsonFilePath | ConvertFrom-Json + $templateParameterNames = $templateJson.parameters.PSObject.Properties.Name + + $templateFileParameters = $templateParameters.Clone() + foreach ($key in $templateParameters.Keys) { + if ($templateParameterNames -notcontains $key) { + Write-Verbose "Removing unnecessary parameter '$key'" + $templateFileParameters.Remove($key) + } + } + + $preDeploymentScript = $templateFile.originalFilePath | Split-Path | Join-Path -ChildPath "$ResourceType-resources-pre.ps1" + if (Test-Path $preDeploymentScript) { + Log "Invoking pre-deployment script '$preDeploymentScript'" + &$preDeploymentScript -ResourceGroupName $ResourceGroupName @PSBoundParameters + } + + $msg = if ($templateFile.jsonFilePath -ne $templateFile.originalFilePath) { + "Deployment template $($templateFile.jsonFilePath) from $($templateFile.originalFilePath) to resource group $($resourceGroup.ResourceGroupName)" + } else { + "Deployment template $($templateFile.jsonFilePath) to resource group $($resourceGroup.ResourceGroupName)" + } + Log $msg + + $deployment = Retry { + New-AzResourceGroupDeployment ` + -Name $BaseName ` + -ResourceGroupName $resourceGroup.ResourceGroupName ` + -TemplateFile $templateFile.jsonFilePath ` + -TemplateParameterObject $templateFileParameters ` + -Force:$Force + } + if ($deployment.ProvisioningState -ne 'Succeeded') { + Write-Host "Deployment '$($deployment.DeploymentName)' has state '$($deployment.ProvisioningState)' with CorrelationId '$($deployment.CorrelationId)'. Exiting..." + Write-Host @' +##################################################### +# For help debugging live test provisioning issues, # +# see http://aka.ms/azsdk/engsys/live-test-help # +##################################################### +'@ + exit 1 + } + + Write-Host "Deployment '$($deployment.DeploymentName)' has CorrelationId '$($deployment.CorrelationId)'" + Write-Host "Successfully deployed template '$($templateFile.jsonFilePath)' to resource group '$($resourceGroup.ResourceGroupName)'" + + $deploymentEnvironmentVariables, $deploymentOutputs = SetDeploymentOutputs ` + -serviceName $serviceName ` + -azContext $context ` + -deployment $deployment ` + -templateFile $templateFile ` + -environmentVariables $EnvironmentVariables + + $postDeploymentScript = $templateFile.originalFilePath | Split-Path | Join-Path -ChildPath "$ResourceType-resources-post.ps1" + if (Test-Path $postDeploymentScript) { + Log "Invoking post-deployment script '$postDeploymentScript'" + &$postDeploymentScript -ResourceGroupName $ResourceGroupName -DeploymentOutputs $deploymentOutputs @PSBoundParameters + } + + if ($templateFile.jsonFilePath.EndsWith('.compiled.json')) { + Write-Verbose "Removing compiled bicep file $($templateFile.jsonFilePath)" + Remove-Item $templateFile.jsonFilePath + } + + Write-Host "Deleting ARM deployment as it may contain secrets. Deployed resources will not be affected." + $null = $deployment | Remove-AzResourceGroupDeployment + } + +} finally { + $exitActions.Invoke() +} + +# Suppress output locally +if ($CI) { + return $deploymentEnvironmentVariables +} + +<# +.SYNOPSIS +Deploys live test resources defined for a service directory to Azure. + +.DESCRIPTION +Deploys live test resouces specified in test-resources.json or test-resources.bicep +files to a new resource group. + +This script searches the directory specified in $ServiceDirectory recursively +for files named test-resources.json or test-resources.bicep. All found test-resources.json +and test-resources.bicep files will be deployed to the test resource group. + +If no test-resources.json or test-resources.bicep files are located the script +exits without making changes to the Azure environment. + +A service principal may optionally be passed to $TestApplicationId and $TestApplicationSecret. +Test resources will grant this service principal access to the created resources. +If no service principal is specified, a new one will be created and assigned the +'Owner' role for the resource group associated with the test resources. + +This script runs in the context of credentials already specified in Connect-AzAccount +or those specified in $ProvisionerApplicationId and $ProvisionerApplicationSecret. + +.PARAMETER BaseName +A name to use in the resource group and passed to the ARM template as 'baseName'. +Limit $BaseName to enough characters to be under limit plus prefixes specified in +the ARM template. See also https://docs.microsoft.com/azure/architecture/best-practices/resource-naming + +Note: The value specified for this parameter will be overriden and generated +by New-TestResources.ps1 if $CI is specified. + +.PARAMETER ResourceGroupName +Set this value to deploy directly to a Resource Group that has already been +created or to create a new resource group with this name. + +If not specified, the $BaseName will be used to generate name for the resource +group that will be created. + +.PARAMETER ServiceDirectory +A directory under 'sdk' in the repository root - optionally with subdirectories +specified - in which to discover ARM templates named 'test-resources.json' and +Bicep templates named 'test-resources.bicep'. This can be an absolute path +or specify parent directories. ServiceDirectory is also used for resource and +environment variable naming. + +.PARAMETER TestResourcesDirectory +An override directory in which to discover ARM templates named 'test-resources.json' and +Bicep templates named 'test-resources.bicep'. This can be an absolute path +or specify parent directories. + +.PARAMETER TestApplicationId +Optional Azure Active Directory Application ID to authenticate the test runner +against deployed resources. Passed to the ARM template as 'testApplicationId'. + +If not specified, a new AAD Application will be created and assigned the 'Owner' +role for the resource group associated with the test resources. No permissions +will be granted to the subscription or other resources. + +For those specifying a Provisioner Application principal as 'ProvisionerApplicationId', +it will need the permission 'Application.ReadWrite.OwnedBy' for the Microsoft Graph API +in order to create the Test Application principal. + +This application is used by the test runner to execute tests against the +live test resources. + +.PARAMETER TestApplicationSecret +Optional service principal secret (password) to authenticate the test runner +against deployed resources. Passed to the ARM template as +'testApplicationSecret'. + +This application is used by the test runner to execute tests against the +live test resources. + +.PARAMETER TestApplicationOid +Service Principal Object ID of the AAD Test Application. This is used to assign +permissions to the AAD application so it can access tested features on the live +test resources (e.g. Role Assignments on resources). It is passed as to the ARM +template as 'testApplicationOid' + +If not specified, an attempt will be made to query it from the Azure Active Directory +tenant. For those specifying a service principal as 'ProvisionerApplicationId', +it will need the permission 'Application.Read.All' for the Microsoft Graph API +in order to query AAD. + +For more information on the relationship between AAD Applications and Service +Principals see: https://docs.microsoft.com/azure/active-directory/develop/app-objects-and-service-principals + +.PARAMETER TenantId +The tenant ID of a service principal when a provisioner is specified. The same +Tenant ID is used for Test Application and Provisioner Application. + +This value is passed to the ARM template as 'tenantId'. + +.PARAMETER SubscriptionId +Optional subscription ID to use for new resources when logging in as a +provisioner. You can also use Set-AzContext if not provisioning. + +If you do not specify a SubscriptionId and are not logged in, one will be +automatically selected for you by the Connect-AzAccount cmdlet. + +Once you are logged in (or were previously), the selected SubscriptionId +will be used for subsequent operations that are specific to a subscription. + +.PARAMETER ProvisionerApplicationId +Optional Application ID of the Azure Active Directory service principal to use for +provisioning the test resources. If not, specified New-TestResources.ps1 uses the +context of the caller to provision. + +If specified, the Provisioner Application principal would benefit from the following +permissions to the Microsoft Graph API: + + - 'Application.Read.All' in order to query AAD to obtain the 'TestApplicaitonOid' + + - 'Application.ReadWrite.OwnedBy' in order to create the Test Application principal + or grant an existing principal ownership of the resource group associated with + the test resources. + +If the provisioner does not have these permissions, it can still be used with +New-TestResources.ps1 by specifying an existing Test Application principal, including +its Object ID, and managing permissions to the resource group manually. + +This value is not passed to the ARM template. + +.PARAMETER ProvisionerApplicationSecret +A service principal secret (password) used to provision test resources when a +provisioner is specified. + +This value is not passed to the ARM template. + +.PARAMETER DeleteAfterHours +Positive integer number of hours from the current time to set the +'DeleteAfter' tag on the created resource group. The computed value is a +timestamp of the form "2020-03-04T09:07:04.3083910Z". + +An optional cleanup process can delete resource groups whose "DeleteAfter" +timestamp is less than the current time. + +This is used for CI automation. + +.PARAMETER Location +Optional location where resources should be created. If left empty, the default +is based on the cloud to which the template is being deployed: + +* AzureCloud -> 'westus' +* AzureUSGovernment -> 'usgovvirginia' +* AzureChinaCloud -> 'chinaeast2' +* Dogfood -> 'westus' + +.PARAMETER Environment +Optional name of the cloud environment. The default is the Azure Public Cloud +('AzureCloud') + +.PARAMETER AdditionalParameters +Optional key-value pairs of parameters to pass to the ARM template(s) and pre-post scripts. + +.PARAMETER ArmTemplateParameters +Optional key-value pairs of parameters to pass to the ARM template(s). + +.PARAMETER EnvironmentVariables +Optional key-value pairs of parameters to set as environment variables to the shell. + +.PARAMETER CI +Indicates the script is run as part of a Continuous Integration / Continuous +Deployment (CI/CD) build (only Azure Pipelines is currently supported). + +.PARAMETER Force +Force creation of resources instead of being prompted. + +.PARAMETER OutFile +Save test environment settings into a .env file next to test resources template. +The contents of the file are protected via the .NET Data Protection API (DPAPI). +This is supported only on Windows. The environment file is scoped to the current +service directory. + +The environment file will be named for the test resources template that it was +generated for. For ARM templates, it will be test-resources.json.env. For +Bicep templates, test-resources.bicep.env. + +.PARAMETER UserAuth +Create the resource group and deploy the template using the signed in user's credentials. +No service principal will be created or used. + +The environment file will be named for the test resources template that it was +generated for. For ARM templates, it will be test-resources.json.env. For +Bicep templates, test-resources.bicep.env. + +.PARAMETER SuppressVsoCommands +By default, the -CI parameter will print out secrets to logs with Azure Pipelines log +commands that cause them to be redacted. For CI environments that don't support this (like +stress test clusters), this flag can be set to $false to avoid printing out these secrets to the logs. + +.EXAMPLE +Connect-AzAccount -Subscription 'REPLACE_WITH_SUBSCRIPTION_ID' +New-TestResources.ps1 keyvault + +Run this in a desktop environment to create a new AAD application and Service Principal +for running live tests against the test resources created. The principal will have ownership +rights to the resource group and the resources that it contains, but no other resources in +the subscription. + +Requires PowerShell 7 to use ConvertFrom-SecureString -AsPlainText or convert +the SecureString to plaintext by another means. + +.EXAMPLE +Connect-AzAccount -Subscription 'REPLACE_WITH_SUBSCRIPTION_ID' +New-TestResources.ps1 ` + -BaseName 'azsdk' ` + -ServiceDirectory 'keyvault' ` + -SubscriptionId 'REPLACE_WITH_SUBSCRIPTION_ID' ` + -ResourceGroupName 'REPLACE_WITH_NAME_FOR_RESOURCE_GROUP' ` + -Location 'eastus' + +Run this in a desktop environment to specify the name and location of the resource +group that test resources are being deployed to. This will also create a new AAD +application and Service Principal for running live tests against the rest resources +created. The principal will have ownership rights to the resource group and the +resources that it contains, but no other resources in the subscription. + +Requires PowerShell 7 to use ConvertFrom-SecureString -AsPlainText or convert +the SecureString to plaintext by another means. + +.EXAMPLE +Connect-AzAccount -Subscription 'REPLACE_WITH_SUBSCRIPTION_ID' +New-TestResources.ps1 ` + -BaseName 'azsdk' ` + -ServiceDirectory 'keyvault' ` + -SubscriptionId 'REPLACE_WITH_SUBSCRIPTION_ID' ` + -ResourceGroupName 'REPLACE_WITH_NAME_FOR_RESOURCE_GROUP' ` + -Location 'eastus' ` + -TestApplicationId 'REPLACE_WITH_TEST_APPLICATION_ID' ` + -TestApplicationSecret 'REPLACE_WITH_TEST_APPLICATION_SECRET' + +Run this in a desktop environment to specify the name and location of the resource +group that test resources are being deployed to. This will grant ownership rights +to the 'TestApplicationId' for the resource group and the resources that it contains, +without altering its existing permissions. + +.EXAMPLE +New-TestResources.ps1 ` + -BaseName 'azsdk' ` + -ServiceDirectory 'keyvault' ` + -SubscriptionId 'REPLACE_WITH_SUBSCRIPTION_ID' ` + -ResourceGroupName 'REPLACE_WITH_NAME_FOR_RESOURCE_GROUP' ` + -Location 'eastus' ` + -ProvisionerApplicationId 'REPLACE_WITH_PROVISIONER_APPLICATION_ID' ` + -ProvisionerApplicationSecret 'REPLACE_WITH_PROVISIONER_APPLICATION_ID' ` + -TestApplicationId 'REPLACE_WITH_TEST_APPLICATION_ID' ` + -TestApplicationOid 'REPLACE_WITH_TEST_APPLICATION_OBJECT_ID' ` + -TestApplicationSecret 'REPLACE_WITH_TEST_APPLICATION_SECRET' + +Run this in a desktop environment to specify the name and location of the resource +group that test resources are being deployed to. The script will be executed in the +context of the 'ProvisionerApplicationId' rather than the caller. + +Depending on the permissions of the Provisioner Application principal, the script may +grant ownership rights 'TestApplicationId' for the resource group and the resources +that it contains, or may emit a message indicating that it was unable to perform the grant. + +For the Provisioner Application principal to perform the grant, it will need the +permission 'Application.ReadWrite.OwnedBy' for the Microsoft Graph API. + +Requires PowerShell 7 to use ConvertFrom-SecureString -AsPlainText or convert +the SecureString to plaintext by another means. + +.EXAMPLE +New-TestResources.ps1 ` + -ServiceDirectory '$(ServiceDirectory)' ` + -TenantId '$(TenantId)' ` + -ProvisionerApplicationId '$(ProvisionerId)' ` + -ProvisionerApplicationSecret '$(ProvisionerSecret)' ` + -TestApplicationId '$(TestAppId)' ` + -TestApplicationSecret '$(TestAppSecret)' ` + -DeleteAfterHours 24 ` + -CI ` + -Force ` + -Verbose + +Run this in an Azure DevOps CI (with approrpiate variables configured) before +executing live tests. The script will output variables as secrets (to enable +log redaction). + +#> diff --git a/eng/common/TestResources/New-TestResources.ps1.md b/eng/common/TestResources/New-TestResources.ps1.md new file mode 100644 index 00000000..b09ba04d --- /dev/null +++ b/eng/common/TestResources/New-TestResources.ps1.md @@ -0,0 +1,727 @@ +--- +external help file: -help.xml +Module Name: +online version: +schema: 2.0.0 +--- + +# New-TestResources.ps1 + +## SYNOPSIS +Deploys live test resources defined for a service directory to Azure. + +## SYNTAX + +### Default (Default) +``` +New-TestResources.ps1 [-BaseName ] [-ResourceGroupName ] [-ServiceDirectory] + [-TestResourcesDirectory ] [-TestApplicationId ] [-TestApplicationSecret ] + [-TestApplicationOid ] [-SubscriptionId ] [-DeleteAfterHours ] [-Location ] + [-Environment ] [-ResourceType ] [-ArmTemplateParameters ] + [-AdditionalParameters ] [-EnvironmentVariables ] [-CI] [-Force] [-OutFile] + [-SuppressVsoCommands] [-UserAuth] [-NewTestResourcesRemainingArguments ] + [-ProgressAction ] [-WhatIf] [-Confirm] [] +``` + +### Provisioner +``` +New-TestResources.ps1 [-BaseName ] [-ResourceGroupName ] [-ServiceDirectory] + [-TestResourcesDirectory ] [-TestApplicationId ] [-TestApplicationSecret ] + [-TestApplicationOid ] -TenantId [-SubscriptionId ] + -ProvisionerApplicationId [-ProvisionerApplicationOid ] + -ProvisionerApplicationSecret [-DeleteAfterHours ] [-Location ] + [-Environment ] [-ResourceType ] [-ArmTemplateParameters ] + [-AdditionalParameters ] [-EnvironmentVariables ] [-CI] [-Force] [-OutFile] + [-SuppressVsoCommands] [-UserAuth] [-NewTestResourcesRemainingArguments ] + [-ProgressAction ] [-WhatIf] [-Confirm] [] +``` + +## DESCRIPTION +Deploys live test resouces specified in test-resources.json or test-resources.bicep +files to a new resource group. + +This script searches the directory specified in $ServiceDirectory recursively +for files named test-resources.json or test-resources.bicep. +All found test-resources.json +and test-resources.bicep files will be deployed to the test resource group. + +If no test-resources.json or test-resources.bicep files are located the script +exits without making changes to the Azure environment. + +A service principal may optionally be passed to $TestApplicationId and $TestApplicationSecret. +Test resources will grant this service principal access to the created resources. +If no service principal is specified, a new one will be created and assigned the +'Owner' role for the resource group associated with the test resources. + +This script runs in the context of credentials already specified in Connect-AzAccount +or those specified in $ProvisionerApplicationId and $ProvisionerApplicationSecret. + +## EXAMPLES + +### EXAMPLE 1 +``` +Connect-AzAccount -Subscription 'REPLACE_WITH_SUBSCRIPTION_ID' +New-TestResources.ps1 keyvault +``` + +Run this in a desktop environment to create a new AAD application and Service Principal +for running live tests against the test resources created. +The principal will have ownership +rights to the resource group and the resources that it contains, but no other resources in +the subscription. + +Requires PowerShell 7 to use ConvertFrom-SecureString -AsPlainText or convert +the SecureString to plaintext by another means. + +### EXAMPLE 2 +``` +Connect-AzAccount -Subscription 'REPLACE_WITH_SUBSCRIPTION_ID' +New-TestResources.ps1 ` + -BaseName 'azsdk' ` + -ServiceDirectory 'keyvault' ` + -SubscriptionId 'REPLACE_WITH_SUBSCRIPTION_ID' ` + -ResourceGroupName 'REPLACE_WITH_NAME_FOR_RESOURCE_GROUP' ` + -Location 'eastus' +``` + +Run this in a desktop environment to specify the name and location of the resource +group that test resources are being deployed to. +This will also create a new AAD +application and Service Principal for running live tests against the rest resources +created. +The principal will have ownership rights to the resource group and the +resources that it contains, but no other resources in the subscription. + +Requires PowerShell 7 to use ConvertFrom-SecureString -AsPlainText or convert +the SecureString to plaintext by another means. + +### EXAMPLE 3 +``` +Connect-AzAccount -Subscription 'REPLACE_WITH_SUBSCRIPTION_ID' +New-TestResources.ps1 ` + -BaseName 'azsdk' ` + -ServiceDirectory 'keyvault' ` + -SubscriptionId 'REPLACE_WITH_SUBSCRIPTION_ID' ` + -ResourceGroupName 'REPLACE_WITH_NAME_FOR_RESOURCE_GROUP' ` + -Location 'eastus' ` + -TestApplicationId 'REPLACE_WITH_TEST_APPLICATION_ID' ` + -TestApplicationSecret 'REPLACE_WITH_TEST_APPLICATION_SECRET' +``` + +Run this in a desktop environment to specify the name and location of the resource +group that test resources are being deployed to. +This will grant ownership rights +to the 'TestApplicationId' for the resource group and the resources that it contains, +without altering its existing permissions. + +### EXAMPLE 4 +``` +New-TestResources.ps1 ` + -BaseName 'azsdk' ` + -ServiceDirectory 'keyvault' ` + -SubscriptionId 'REPLACE_WITH_SUBSCRIPTION_ID' ` + -ResourceGroupName 'REPLACE_WITH_NAME_FOR_RESOURCE_GROUP' ` + -Location 'eastus' ` + -ProvisionerApplicationId 'REPLACE_WITH_PROVISIONER_APPLICATION_ID' ` + -ProvisionerApplicationSecret 'REPLACE_WITH_PROVISIONER_APPLICATION_ID' ` + -TestApplicationId 'REPLACE_WITH_TEST_APPLICATION_ID' ` + -TestApplicationOid 'REPLACE_WITH_TEST_APPLICATION_OBJECT_ID' ` + -TestApplicationSecret 'REPLACE_WITH_TEST_APPLICATION_SECRET' +``` + +Run this in a desktop environment to specify the name and location of the resource +group that test resources are being deployed to. +The script will be executed in the +context of the 'ProvisionerApplicationId' rather than the caller. + +Depending on the permissions of the Provisioner Application principal, the script may +grant ownership rights 'TestApplicationId' for the resource group and the resources +that it contains, or may emit a message indicating that it was unable to perform the grant. + +For the Provisioner Application principal to perform the grant, it will need the +permission 'Application.ReadWrite.OwnedBy' for the Microsoft Graph API. + +Requires PowerShell 7 to use ConvertFrom-SecureString -AsPlainText or convert +the SecureString to plaintext by another means. + +### EXAMPLE 5 +``` +New-TestResources.ps1 ` + -ServiceDirectory '$(ServiceDirectory)' ` + -TenantId '$(TenantId)' ` + -ProvisionerApplicationId '$(ProvisionerId)' ` + -ProvisionerApplicationSecret '$(ProvisionerSecret)' ` + -TestApplicationId '$(TestAppId)' ` + -TestApplicationSecret '$(TestAppSecret)' ` + -DeleteAfterHours 24 ` + -CI ` + -Force ` + -Verbose +``` + +Run this in an Azure DevOps CI (with approrpiate variables configured) before +executing live tests. +The script will output variables as secrets (to enable +log redaction). + +## PARAMETERS + +### -BaseName +A name to use in the resource group and passed to the ARM template as 'baseName'. +Limit $BaseName to enough characters to be under limit plus prefixes specified in +the ARM template. +See also https://docs.microsoft.com/azure/architecture/best-practices/resource-naming + +Note: The value specified for this parameter will be overriden and generated +by New-TestResources.ps1 if $CI is specified. + +```yaml +Type: String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ResourceGroupName +Set this value to deploy directly to a Resource Group that has already been +created or to create a new resource group with this name. + +If not specified, the $BaseName will be used to generate name for the resource +group that will be created. + +```yaml +Type: String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ServiceDirectory +A directory under 'sdk' in the repository root - optionally with subdirectories +specified - in which to discover ARM templates named 'test-resources.json' and +Bicep templates named 'test-resources.bicep'. +This can be an absolute path +or specify parent directories. +ServiceDirectory is also used for resource and +environment variable naming. + +```yaml +Type: String +Parameter Sets: (All) +Aliases: + +Required: True +Position: 1 +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -TestResourcesDirectory +An override directory in which to discover ARM templates named 'test-resources.json' and +Bicep templates named 'test-resources.bicep'. +This can be an absolute path +or specify parent directories. + +```yaml +Type: String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -TestApplicationId +Optional Azure Active Directory Application ID to authenticate the test runner +against deployed resources. +Passed to the ARM template as 'testApplicationId'. + +If not specified, a new AAD Application will be created and assigned the 'Owner' +role for the resource group associated with the test resources. +No permissions +will be granted to the subscription or other resources. + +For those specifying a Provisioner Application principal as 'ProvisionerApplicationId', +it will need the permission 'Application.ReadWrite.OwnedBy' for the Microsoft Graph API +in order to create the Test Application principal. + +This application is used by the test runner to execute tests against the +live test resources. + +```yaml +Type: String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -TestApplicationSecret +Optional service principal secret (password) to authenticate the test runner +against deployed resources. +Passed to the ARM template as +'testApplicationSecret'. + +This application is used by the test runner to execute tests against the +live test resources. + +```yaml +Type: String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -TestApplicationOid +Service Principal Object ID of the AAD Test Application. +This is used to assign +permissions to the AAD application so it can access tested features on the live +test resources (e.g. +Role Assignments on resources). +It is passed as to the ARM +template as 'testApplicationOid' + +If not specified, an attempt will be made to query it from the Azure Active Directory +tenant. +For those specifying a service principal as 'ProvisionerApplicationId', +it will need the permission 'Application.Read.All' for the Microsoft Graph API +in order to query AAD. + +For more information on the relationship between AAD Applications and Service +Principals see: https://docs.microsoft.com/azure/active-directory/develop/app-objects-and-service-principals + +```yaml +Type: String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -TenantId +The tenant ID of a service principal when a provisioner is specified. +The same +Tenant ID is used for Test Application and Provisioner Application. + +This value is passed to the ARM template as 'tenantId'. + +```yaml +Type: String +Parameter Sets: Provisioner +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -SubscriptionId +Optional subscription ID to use for new resources when logging in as a +provisioner. +You can also use Set-AzContext if not provisioning. + +If you do not specify a SubscriptionId and are not logged in, one will be +automatically selected for you by the Connect-AzAccount cmdlet. + +Once you are logged in (or were previously), the selected SubscriptionId +will be used for subsequent operations that are specific to a subscription. + +```yaml +Type: String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ProvisionerApplicationId +Optional Application ID of the Azure Active Directory service principal to use for +provisioning the test resources. +If not, specified New-TestResources.ps1 uses the +context of the caller to provision. + +If specified, the Provisioner Application principal would benefit from the following +permissions to the Microsoft Graph API: + + - 'Application.Read.All' in order to query AAD to obtain the 'TestApplicaitonOid' + + - 'Application.ReadWrite.OwnedBy' in order to create the Test Application principal + or grant an existing principal ownership of the resource group associated with + the test resources. + +If the provisioner does not have these permissions, it can still be used with +New-TestResources.ps1 by specifying an existing Test Application principal, including +its Object ID, and managing permissions to the resource group manually. + +This value is not passed to the ARM template. + +```yaml +Type: String +Parameter Sets: Provisioner +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ProvisionerApplicationOid +{{ Fill ProvisionerApplicationOid Description }} + +```yaml +Type: String +Parameter Sets: Provisioner +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ProvisionerApplicationSecret +A service principal secret (password) used to provision test resources when a +provisioner is specified. + +This value is not passed to the ARM template. + +```yaml +Type: String +Parameter Sets: Provisioner +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -DeleteAfterHours +Positive integer number of hours from the current time to set the +'DeleteAfter' tag on the created resource group. +The computed value is a +timestamp of the form "2020-03-04T09:07:04.3083910Z". + +An optional cleanup process can delete resource groups whose "DeleteAfter" +timestamp is less than the current time. + +This is used for CI automation. + +```yaml +Type: Int32 +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: 120 +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Location +Optional location where resources should be created. +If left empty, the default +is based on the cloud to which the template is being deployed: + +* AzureCloud -\> 'westus' +* AzureUSGovernment -\> 'usgovvirginia' +* AzureChinaCloud -\> 'chinaeast2' +* Dogfood -\> 'westus' + +```yaml +Type: String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Environment +Optional name of the cloud environment. +The default is the Azure Public Cloud +('AzureCloud') + +```yaml +Type: String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: AzureCloud +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ResourceType +{{ Fill ResourceType Description }} + +```yaml +Type: String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: Test +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ArmTemplateParameters +Optional key-value pairs of parameters to pass to the ARM template(s). + +```yaml +Type: Hashtable +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -AdditionalParameters +Optional key-value pairs of parameters to pass to the ARM template(s) and pre-post scripts. + +```yaml +Type: Hashtable +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -EnvironmentVariables +Optional key-value pairs of parameters to set as environment variables to the shell. + +```yaml +Type: Hashtable +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: @{} +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -CI +Indicates the script is run as part of a Continuous Integration / Continuous +Deployment (CI/CD) build (only Azure Pipelines is currently supported). + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: ($null -ne $env:SYSTEM_TEAMPROJECTID) +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Force +Force creation of resources instead of being prompted. + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: False +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -OutFile +Save test environment settings into a .env file next to test resources template. +The contents of the file are protected via the .NET Data Protection API (DPAPI). +This is supported only on Windows. +The environment file is scoped to the current +service directory. + +The environment file will be named for the test resources template that it was +generated for. +For ARM templates, it will be test-resources.json.env. +For +Bicep templates, test-resources.bicep.env. + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: False +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -SuppressVsoCommands +By default, the -CI parameter will print out secrets to logs with Azure Pipelines log +commands that cause them to be redacted. +For CI environments that don't support this (like +stress test clusters), this flag can be set to $false to avoid printing out these secrets to the logs. + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: ($null -eq $env:SYSTEM_TEAMPROJECTID) +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -UserAuth +Create the resource group and deploy the template using the signed in user's credentials. +No service principal will be created or used. + +The environment file will be named for the test resources template that it was +generated for. +For ARM templates, it will be test-resources.json.env. +For +Bicep templates, test-resources.bicep.env. + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: False +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -NewTestResourcesRemainingArguments +Captures any arguments not declared here (no parameter errors) +This enables backwards compatibility with old script versions in +hotfix branches if and when the dynamic subscription configuration +secrets get updated to add new parameters. + +```yaml +Type: Object +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -WhatIf +Shows what would happen if the cmdlet runs. +The cmdlet is not run. + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: wi + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Confirm +Prompts you for confirmation before running the cmdlet. + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: cf + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ProgressAction +{{ Fill ProgressAction Description }} + +```yaml +Type: ActionPreference +Parameter Sets: (All) +Aliases: proga + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### CommonParameters +This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see [about_CommonParameters](https://go.microsoft.com/fwlink/?LinkID=113216). + +## INPUTS + +## OUTPUTS + +## NOTES + +## RELATED LINKS diff --git a/eng/common/TestResources/README.md b/eng/common/TestResources/README.md new file mode 100644 index 00000000..b63307e2 --- /dev/null +++ b/eng/common/TestResources/README.md @@ -0,0 +1,192 @@ +# Live Test Resource Management + +Running and recording live tests often requires first creating some resources +in Azure. Service directories that include a `test-resources.json` or `test-resources.bicep` +file require running [New-TestResources.ps1][] to create these resources and output +environment variables you must set. + +The following scripts can be used both in on your desktop for developer +scenarios as well as on hosted agents for continuous integration testing. + +* [New-TestResources.ps1][] - Creates new test resources for a given service. +* [Remove-TestResources.ps1][] - Deletes previously created resources. + +## Prerequisites + +1. Install [PowerShell][] version 7.0 or newer. +2. Install the [Azure PowerShell][PowerShellAz]. + +## On the Desktop + +To set up your Azure account to run live tests, you'll need to log into Azure, +and create the resources defined in your `test-resources.json` or `test-resources.bicep` +template as shown in the following example using Azure Key Vault. The script will create +a service principal automatically, or you may create a service principal that can be reused +subsequently. + +Note that `-Subscription` is an optional parameter but recommended if your account +is a member of multiple subscriptions. If you didn't specify it when logging in, +you should select your desired subscription using `Select-AzSubscription`. The +default can be saved using `Set-AzDefault` for future sessions. + +```powershell +Connect-AzAccount -Subscription 'YOUR SUBSCRIPTION ID' +eng\common\TestResources\New-TestResources.ps1 keyvault +``` + +The `OutFile` switch will be set by default if you are running this for a .NET project on Windows. +This will save test environment settings into a `test-resources.json.env` file next to `test-resources.json` +or a `test-resources.bicep.env` file next to `test-resources.bicep`. The file is protected via DPAPI. +The environment file would be scoped to the current repository directory and avoids the need to +set environment variables or restart your IDE to recognize them. + +Along with some log messages, this will output environment variables based on +your current shell like in the following example: + +```powershell +${env:KEYVAULT_TENANT_ID} = '<>' +${env:KEYVAULT_CLIENT_ID} = '<>' +${env:KEYVAULT_CLIENT_SECRET} = '<>' +${env:KEYVAULT_SUBSCRIPTION_ID} = 'YOUR SUBSCRIPTION ID' +${env:KEYVAULT_RESOURCE_GROUP} = 'rg-myusername' +${env:KEYVAULT_LOCATION} = 'westus' +${env:KEYVAULT_SKU} = 'premium' +${env:AZURE_KEYVAULT_URL} = '<>' +``` + +For security reasons we do not set these environment variables automatically +for either the current process or persistently for future sessions. You must +do that yourself based on your current platform and shell. + +If your current shell was detected properly, you should be able to copy and +paste the output directly in your terminal and add to your profile script. +For example, in PowerShell on Windows you can copy the output above and paste +it back into the terminal to set those environment variables for the current +process. To persist these variables for future terminal sessions or for +applications started outside the terminal, you could copy and paste the +following commands: + +```powershell +setx KEYVAULT_TENANT_ID ${env:KEYVAULT_TENANT_ID} +setx KEYVAULT_CLIENT_ID ${env:KEYVAULT_CLIENT_ID} +setx KEYVAULT_CLIENT_SECRET ${env:KEYVAULT_CLIENT_SECRET} +setx KEYVAULT_SUBSCRIPTION_ID ${env:KEYVAULT_SUBSCRIPTION_ID} +setx KEYVAULT_RESOURCE_GROUP ${env:KEYVAULT_RESOURCE_GROUP} +setx KEYVAULT_LOCATION ${env:KEYVAULT_LOCATION} +setx KEYVAULT_SKU ${env:KEYVAULT_SKU} +setx AZURE_KEYVAULT_URL ${env:AZURE_KEYVAULT_URL} +``` + +### Pre- and Post- Scripts + +Sometimes creating test resources requires either some work to be done prior to or after the main test-resources.json script is executed. +For these scenarios a `test-resources-pre.ps1` or `test-resources-post.ps1`, respectively, can be created in the same folder as the `test-resources.json` file. + +For example, it may be necessary to create artifacts prior to provisioning the actual resource, such as a certificate. +Typically the created artifact will need to be passed to `test-resources.json` to be used in the ARM template or as output (or both). + +Below is an example of how `$templateFileParameters` can be used to pass data from the `pre-` script to `test-resources.json`. + +**Snippet from `test-resources-pre.ps1`** +```powershell +Import-Module -Name ./eng/common/scripts/X509Certificate2 +$cert = New-X509Certificate2 -SubjectName 'E=opensource@microsoft.com, CN=Azure SDK, OU=Azure SDK, O=Microsoft, L=Frisco, S=TX, C=US' -ValidDays 3652 +# Create new entries in $templateFileParameters +$templateFileParameters['ConfidentialLedgerPrincipalPEM'] = Format-X509Certificate2 -Certificate $cert +$templateFileParameters['ConfidentialLedgerPrincipalPEMPK'] = Format-X509Certificate2 -Type Pkcs8 -Certificate $cert +``` + +**Snippet from the corresponding `test-resources.json`.** + +Note that the values present in `$templateFileParameters` will map to parameters of the same name. +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "_comment": "Other required parameters would go here... (this is not part of the actual test-resources.json)", + "ConfidentialLedgerPrincipalPEM": { + "type": "string", + "metadata": { + "description": "The certificate to configure as a certBasedSecurityPrincipal." + } + }, + "ConfidentialLedgerPrincipalPEMPK": { + "type": "string", + "metadata": { + "description": "The certificate to configure as a certBasedSecurityPrincipal." + } + } + }, +} +``` + +### Cleaning up Resources + +By default, resource groups are tagged with a `DeleteAfter` value and date according to the default or specified +value for the `-DeleteAfterHours` switch. You can use this tag in scheduled jobs to remove older resources based +on that date. + +If you are not ready for the resources to be deleted, you can update the resource group by running [Update-TestResources.ps1][]: + +```powershell +Update-TestResources.ps1 keyvault +``` + +This will extend the expiration time by the default value (e.g. 48 hours) from now. + +Alternatively, after running or recording live tests, if you do not plan on further testing +you can immediately remove the test resources you created above by running [Remove-TestResources.ps1][]: + +```powershell +Remove-TestResources.ps1 keyvault -Force +``` + +If you persisted environment variables, you should also remove those as well. + +### Passing Additional Arguments + +Some test-resources.json templates utilize the `AdditionalParameters` parameter to control additional resource configuration options. For example: + +```powershell +New-TestResources.ps1 keyvault -AdditionalParameters @{enableHsm = $true} +``` + +## In CI + +Test pipelines should include deploy-test-resources.yml and +remove-test-resources.yml like in the following examples: + +```yml +- template: /eng/common/TestResources/deploy-test-resources.yml + parameters: + ServiceDirectory: '${{ parameters.ServiceDirectory }}' + +# Run tests + +- template: /eng/common/TestResources/remove-test-resources.yml +``` + +Be sure to link the **Secrets for Resource Provisioner** variable group +into the test pipeline for these scripts to work. + +## Documentation + +To regenerate documentation for scripts within this directory, you can install +[platyPS][] and run it like in the following example: + +```powershell +Install-Module platyPS -Scope CurrentUser -Force +New-MarkdownHelp -Command .\New-TestResources.ps1 -OutputFolder . -Force +``` + +After the markdown files are generated, please make sure all "http" URIs use "https". + +PowerShell markdown documentation created with [platyPS][]. + + [New-TestResources.ps1]: https://aka.ms/azsdk/tools/New-TestResources + [Update-TestResources.ps1]: https://aka.ms/azsdk/tools/Update-TestResources + [Remove-TestResources.ps1]: https://aka.ms/azsdk/tools/Remove-TestResources + [PowerShell]: https://github.com/PowerShell/PowerShell + [PowerShellAz]: https://docs.microsoft.com/powershell/azure/install-az-ps + [platyPS]: https://github.com/PowerShell/platyPS diff --git a/eng/common/TestResources/Remove-TestResources.cmd b/eng/common/TestResources/Remove-TestResources.cmd new file mode 100644 index 00000000..94b0c1f2 --- /dev/null +++ b/eng/common/TestResources/Remove-TestResources.cmd @@ -0,0 +1,17 @@ +@echo off + +REM Copyright (c) Microsoft Corporation. All rights reserved. +REM Licensed under the MIT License. + +setlocal + +for /f "usebackq delims=" %%i in (`where pwsh 2^>nul`) do ( + set _cmd=%%i +) + +if "%_cmd%"=="" ( + echo Error: PowerShell not found. Please visit https://github.com/powershell/powershell for install instructions. + exit /b 2 +) + +call "%_cmd%" -NoLogo -NoProfile -File "%~dpn0.ps1" %* diff --git a/eng/common/TestResources/Remove-TestResources.ps1 b/eng/common/TestResources/Remove-TestResources.ps1 new file mode 100644 index 00000000..4e2b52c6 --- /dev/null +++ b/eng/common/TestResources/Remove-TestResources.ps1 @@ -0,0 +1,324 @@ +#!/usr/bin/env pwsh + +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +#Requires -Version 6.0 +#Requires -PSEdition Core +#Requires -Modules @{ModuleName='Az.Accounts'; ModuleVersion='1.6.4'} +#Requires -Modules @{ModuleName='Az.Resources'; ModuleVersion='1.8.0'} + +[CmdletBinding(DefaultParameterSetName = 'Default', SupportsShouldProcess = $true, ConfirmImpact = 'Medium')] +param ( + # Limit $BaseName to enough characters to be under limit plus prefixes, and https://docs.microsoft.com/azure/architecture/best-practices/resource-naming. + [Parameter(ParameterSetName = 'Default')] + [Parameter(ParameterSetName = 'Default+Provisioner', Mandatory = $true, Position = 0)] + [ValidatePattern('^[-a-zA-Z0-9\.\(\)_]{0,80}(?<=[a-zA-Z0-9\(\)])$')] + [string] $BaseName, + + [Parameter(ParameterSetName = 'ResourceGroup')] + [Parameter(ParameterSetName = 'ResourceGroup+Provisioner')] + [string] $ResourceGroupName, + + [Parameter(ParameterSetName = 'Default+Provisioner', Mandatory = $true)] + [Parameter(ParameterSetName = 'ResourceGroup+Provisioner', Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] $TenantId, + + [Parameter()] + [ValidatePattern('^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$')] + [string] $SubscriptionId, + + [Parameter(ParameterSetName = 'Default+Provisioner', Mandatory = $true)] + [Parameter(ParameterSetName = 'ResourceGroup+Provisioner', Mandatory = $true)] + [ValidatePattern('^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$')] + [string] $ProvisionerApplicationId, + + [Parameter(ParameterSetName = 'Default+Provisioner', Mandatory = $true)] + [Parameter(ParameterSetName = 'ResourceGroup+Provisioner', Mandatory = $true)] + [string] $ProvisionerApplicationSecret, + + [Parameter(ParameterSetName = 'Default', Position = 0)] + [Parameter(ParameterSetName = 'Default+Provisioner')] + [Parameter(ParameterSetName = 'ResourceGroup')] + [Parameter(ParameterSetName = 'ResourceGroup+Provisioner')] + [string] $ServiceDirectory, + + [Parameter()] + [ValidateSet('AzureCloud', 'AzureUSGovernment', 'AzureChinaCloud', 'Dogfood')] + [string] $Environment = 'AzureCloud', + + [Parameter(ParameterSetName = 'ResourceGroup')] + [Parameter(ParameterSetName = 'ResourceGroup+Provisioner')] + [switch] $CI, + + [Parameter()] + [ValidateSet('test', 'perf')] + [string] $ResourceType = 'test', + + [Parameter()] + [switch] $Force, + + # Captures any arguments not declared here (no parameter errors) + [Parameter(ValueFromRemainingArguments = $true)] + $RemoveTestResourcesRemainingArguments +) + +# By default stop for any error. +if (!$PSBoundParameters.ContainsKey('ErrorAction')) { + $ErrorActionPreference = 'Stop' +} + +# Support actions to invoke on exit. +$exitActions = @({ + if ($exitActions.Count -gt 1) { + Write-Verbose 'Running registered exit actions.' + } +}) + +trap { + # Like using try..finally in PowerShell, but without keeping track of more braces or tabbing content. + $exitActions.Invoke() +} + +. $PSScriptRoot/SubConfig-Helpers.ps1 +# Source helpers to purge resources. +. "$PSScriptRoot\..\scripts\Helpers\Resource-Helpers.ps1" + +function Log($Message) { + Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message) +} + +function Retry([scriptblock] $Action, [int] $Attempts = 5) { + $attempt = 0 + $sleep = 5 + + while ($attempt -lt $Attempts) { + try { + $attempt++ + return $Action.Invoke() + } catch { + if ($attempt -lt $Attempts) { + $sleep *= 2 + + Write-Warning "Attempt $attempt failed: $_. Trying again in $sleep seconds..." + Start-Sleep -Seconds $sleep + } else { + Write-Error -ErrorRecord $_ + } + } + } +} + +if ($ProvisionerApplicationId) { + $null = Disable-AzContextAutosave -Scope Process + + Log "Logging into service principal '$ProvisionerApplicationId'" + $provisionerSecret = ConvertTo-SecureString -String $ProvisionerApplicationSecret -AsPlainText -Force + $provisionerCredential = [System.Management.Automation.PSCredential]::new($ProvisionerApplicationId, $provisionerSecret) + + # Use the given subscription ID if provided. + $subscriptionArgs = if ($SubscriptionId) { + @{SubscriptionId = $SubscriptionId} + } + + $provisionerAccount = Retry { + Connect-AzAccount -Force:$Force -Tenant $TenantId -Credential $provisionerCredential -ServicePrincipal -Environment $Environment @subscriptionArgs + } + + $exitActions += { + Write-Verbose "Logging out of service principal '$($provisionerAccount.Context.Account)'" + $null = Disconnect-AzAccount -AzureContext $provisionerAccount.Context + } +} + +$context = Get-AzContext + +if (!$ResourceGroupName) { + if ($CI) { + if (!$ServiceDirectory) { + Write-Warning "ServiceDirectory parameter is empty, nothing to remove" + exit 0 + } + $envVarName = (BuildServiceDirectoryPrefix (GetServiceLeafDirectoryName $ServiceDirectory)) + "RESOURCE_GROUP" + $ResourceGroupName = [Environment]::GetEnvironmentVariable($envVarName) + if (!$ResourceGroupName) { + Write-Error "Could not find resource group name environment variable '$envVarName'. This is likely due to an earlier failure in the 'Deploy Test Resources' step above." + exit 0 + } + } else { + $serviceName = GetServiceLeafDirectoryName $ServiceDirectory + $BaseName, $ResourceGroupName = GetBaseAndResourceGroupNames ` + -baseNameDefault $BaseName ` + -resourceGroupNameDefault $ResourceGroupName ` + -user (GetUserName) ` + -serviceDirectoryName $serviceName ` + -CI $CI + } +} + +# If no subscription was specified, try to select the Azure SDK Developer Playground subscription. +# Ignore errors to leave the automatically selected subscription. +if ($SubscriptionId) { + $currentSubcriptionId = $context.Subscription.Id + if ($currentSubcriptionId -ne $SubscriptionId) { + Log "Selecting subscription '$SubscriptionId'" + $null = Select-AzSubscription -Subscription $SubscriptionId + + $exitActions += { + Log "Selecting previous subscription '$currentSubcriptionId'" + $null = Select-AzSubscription -Subscription $currentSubcriptionId + } + + # Update the context. + $context = Get-AzContext + } +} else { + Log "Attempting to select subscription 'Azure SDK Developer Playground (faa080af-c1d8-40ad-9cce-e1a450ca5b57)'" + $null = Select-AzSubscription -Subscription 'faa080af-c1d8-40ad-9cce-e1a450ca5b57' -ErrorAction Ignore + + # Update the context. + $context = Get-AzContext + + $SubscriptionId = $context.Subscription.Id + $PSBoundParameters['SubscriptionId'] = $SubscriptionId +} + +# Use cache of well-known team subs without having to be authenticated. +$wellKnownSubscriptions = @{ + 'faa080af-c1d8-40ad-9cce-e1a450ca5b57' = 'Azure SDK Developer Playground' + 'a18897a6-7e44-457d-9260-f2854c0aca42' = 'Azure SDK Engineering System' + '2cd617ea-1866-46b1-90e3-fffb087ebf9b' = 'Azure SDK Test Resources' +} + +# Print which subscription is currently selected. +$subscriptionName = $context.Subscription.Id +if ($wellKnownSubscriptions.ContainsKey($subscriptionName)) { + $subscriptionName = '{0} ({1})' -f $wellKnownSubscriptions[$subscriptionName], $subscriptionName +} + +Log "Selected subscription '$subscriptionName'" + +if ($ServiceDirectory) { + $root = [System.IO.Path]::Combine("$PSScriptRoot/../../../sdk", $ServiceDirectory) | Resolve-Path + $preRemovalScript = Join-Path -Path $root -ChildPath "remove-$ResourceType-resources-pre.ps1" + if (Test-Path $preRemovalScript) { + Log "Invoking pre resource removal script '$preRemovalScript'" + + if (!$PSCmdlet.ParameterSetName.StartsWith('ResourceGroup')) { + $PSBoundParameters.Add('ResourceGroupName', $ResourceGroupName); + } + + &$preRemovalScript @PSBoundParameters + } + + # Make sure environment files from New-TestResources -OutFile are removed. + Get-ChildItem -Path $root -Filter "$ResourceType-resources.json.env" -Recurse | Remove-Item -Force:$Force +} + +$verifyDeleteScript = { + try { + $group = Get-AzResourceGroup -name $ResourceGroupName + } catch { + if ($_.ToString().Contains("Provided resource group does not exist")) { + Write-Verbose "Resource group '$ResourceGroupName' not found. Continuing..." + return + } + throw $_ + } + + if ($group.ProvisioningState -ne "Deleting") + { + throw "Resource group is in '$($group.ProvisioningState)' state, expected 'Deleting'" + } +} + +# Get any resources that can be purged after the resource group is deleted coerced into a collection even if empty. +$purgeableResources = Get-PurgeableGroupResources $ResourceGroupName + +Log "Deleting resource group '$ResourceGroupName'" +if ($Force -and !$purgeableResources) { + Remove-AzResourceGroup -Name "$ResourceGroupName" -Force:$Force -AsJob + Write-Verbose "Running background job to delete resource group '$ResourceGroupName'" + + Retry $verifyDeleteScript 3 +} else { + # Don't swallow interactive confirmation when Force is false + Remove-AzResourceGroup -Name "$ResourceGroupName" -Force:$Force +} + +# Now purge the resources that should have been deleted with the resource group. +Remove-PurgeableResources $purgeableResources + +$exitActions.Invoke() + +<# +.SYNOPSIS +Deletes the resource group deployed for a service directory from Azure. + +.DESCRIPTION +Removes a resource group and all its resources previously deployed using +New-TestResources.ps1. +If you are not currently logged into an account in the Az PowerShell module, +you will be asked to log in with Connect-AzAccount. Alternatively, you (or a +build pipeline) can pass $ProvisionerApplicationId and +$ProvisionerApplicationSecret to authenticate a service principal with access to +create resources. + +.PARAMETER BaseName +A name to use in the resource group and passed to the ARM template as 'baseName'. +This will delete the resource group named 'rg-' + +.PARAMETER ResourceGroupName +The name of the resource group to delete. + +.PARAMETER TenantId +The tenant ID of a service principal when a provisioner is specified. + +.PARAMETER SubscriptionId +Optional subscription ID to use when deleting resources when logging in as a +provisioner. You can also use Set-AzContext if not provisioning. + +If you do not specify a SubscriptionId and are not logged in, one will be +automatically selected for you by the Connect-AzAccount cmdlet. + +Once you are logged in (or were previously), the selected SubscriptionId +will be used for subsequent operations that are specific to a subscription. + +.PARAMETER ProvisionerApplicationId +A service principal ID to provision test resources when a provisioner is specified. + +.PARAMETER ProvisionerApplicationSecret +A service principal secret (password) to provision test resources when a provisioner is specified. + +.PARAMETER ServiceDirectory +A directory under 'sdk' in the repository root - optionally with subdirectories +specified - in which to discover pre removal script named 'remove-test-resources-pre.json'. + +.PARAMETER Environment +Name of the cloud environment. The default is the Azure Public Cloud +('PublicCloud') + +.PARAMETER CI +Run script in CI mode. Infers various environment variable names based on CI convention. + +.PARAMETER Force +Force removal of resource group without asking for user confirmation + +.EXAMPLE +Remove-TestResources.ps1 keyvault -Force +Use the currently logged-in account to delete the resources created for Key Vault testing. + +.EXAMPLE +Remove-TestResources.ps1 ` + -ResourceGroupName "${env:AZURE_RESOURCEGROUP_NAME}" ` + -TenantId '$(TenantId)' ` + -ProvisionerApplicationId '$(AppId)' ` + -ProvisionerApplicationSecret '$(AppSecret)' ` + -Force ` + -Verbose ` +When run in the context of an Azure DevOps pipeline, this script removes the +resource group whose name is stored in the environment variable +AZURE_RESOURCEGROUP_NAME. + +#> diff --git a/eng/common/TestResources/Remove-TestResources.ps1.md b/eng/common/TestResources/Remove-TestResources.ps1.md new file mode 100644 index 00000000..f6bedada --- /dev/null +++ b/eng/common/TestResources/Remove-TestResources.ps1.md @@ -0,0 +1,308 @@ +--- +external help file: -help.xml +Module Name: +online version: +schema: 2.0.0 +--- + +# Remove-TestResources.ps1 + +## SYNOPSIS +Deletes the resource group deployed for a service directory from Azure. + +## SYNTAX + +### Default (Default) +``` +Remove-TestResources.ps1 [-BaseName ] [-SubscriptionId ] [-ServiceDirectory] + [-Environment ] [-Force] [-RemoveTestResourcesRemainingArguments ] [-WhatIf] [-Confirm] + [] +``` + +### Default+Provisioner +``` +Remove-TestResources.ps1 -BaseName -TenantId [-SubscriptionId ] + -ProvisionerApplicationId -ProvisionerApplicationSecret [[-ServiceDirectory] ] + [-Environment ] [-Force] [-RemoveTestResourcesRemainingArguments ] [-WhatIf] [-Confirm] + [] +``` + +### ResourceGroup+Provisioner +``` +Remove-TestResources.ps1 -ResourceGroupName -TenantId [-SubscriptionId ] + -ProvisionerApplicationId -ProvisionerApplicationSecret [[-ServiceDirectory] ] + [-Environment ] [-CI] [-Force] [-RemoveTestResourcesRemainingArguments ] [-WhatIf] [-Confirm] + [] +``` + +### ResourceGroup +``` +Remove-TestResources.ps1 -ResourceGroupName [-SubscriptionId ] [[-ServiceDirectory] ] + [-Environment ] [-CI] [-Force] [-RemoveTestResourcesRemainingArguments ] [-WhatIf] [-Confirm] + [] +``` + +## DESCRIPTION +Removes a resource group and all its resources previously deployed using +New-TestResources.ps1. +If you are not currently logged into an account in the Az PowerShell module, +you will be asked to log in with Connect-AzAccount. +Alternatively, you (or a +build pipeline) can pass $ProvisionerApplicationId and +$ProvisionerApplicationSecret to authenticate a service principal with access to +create resources. + +## EXAMPLES + +### EXAMPLE 1 +``` +Remove-TestResources.ps1 keyvault -Force +Use the currently logged-in account to delete the resources created for Key Vault testing. +``` + +### EXAMPLE 2 +``` +Remove-TestResources.ps1 ` + -ResourceGroupName "${env:AZURE_RESOURCEGROUP_NAME}" ` + -TenantId '$(TenantId)' ` + -ProvisionerApplicationId '$(AppId)' ` + -ProvisionerApplicationSecret '$(AppSecret)' ` + -Force ` + -Verbose ` +When run in the context of an Azure DevOps pipeline, this script removes the +resource group whose name is stored in the environment variable +AZURE_RESOURCEGROUP_NAME. +``` + +## PARAMETERS + +### -BaseName +A name to use in the resource group and passed to the ARM template as 'baseName'. +This will delete the resource group named 'rg-\' + +```yaml +Type: String +Parameter Sets: Default +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +```yaml +Type: String +Parameter Sets: Default+Provisioner +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ResourceGroupName +The name of the resource group to delete. + +```yaml +Type: String +Parameter Sets: ResourceGroup+Provisioner, ResourceGroup +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -TenantId +The tenant ID of a service principal when a provisioner is specified. + +```yaml +Type: String +Parameter Sets: Default+Provisioner, ResourceGroup+Provisioner +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -SubscriptionId +Optional subscription ID to use when deleting resources when logging in as a +provisioner. +You can also use Set-AzContext if not provisioning. + +If you do not specify a SubscriptionId and are not logged in, one will be +automatically selected for you by the Connect-AzAccount cmdlet. + +Once you are logged in (or were previously), the selected SubscriptionId +will be used for subsequent operations that are specific to a subscription. + +```yaml +Type: String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ProvisionerApplicationId +A service principal ID to provision test resources when a provisioner is specified. + +```yaml +Type: String +Parameter Sets: Default+Provisioner, ResourceGroup+Provisioner +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ProvisionerApplicationSecret +A service principal secret (password) to provision test resources when a provisioner is specified. + +```yaml +Type: String +Parameter Sets: Default+Provisioner, ResourceGroup+Provisioner +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ServiceDirectory +A directory under 'sdk' in the repository root - optionally with subdirectories +specified - in which to discover pre removal script named 'remove-test-resources-pre.json'. + +```yaml +Type: String +Parameter Sets: Default +Aliases: + +Required: True +Position: 1 +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +```yaml +Type: String +Parameter Sets: Default+Provisioner, ResourceGroup+Provisioner, ResourceGroup +Aliases: + +Required: False +Position: 1 +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Environment +Name of the cloud environment. +The default is the Azure Public Cloud +('PublicCloud') + +```yaml +Type: String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: AzureCloud +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -CI +Run script in CI mode. Infers various environment variable names based on CI convention. + +### -Force +Force removal of resource group without asking for user confirmation + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: False +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -RemoveTestResourcesRemainingArguments +Captures any arguments not declared here (no parameter errors) + +```yaml +Type: Object +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -WhatIf +Shows what would happen if the cmdlet runs. +The cmdlet is not run. + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: wi + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Confirm +Prompts you for confirmation before running the cmdlet. + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: cf + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### CommonParameters +This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see [about_CommonParameters](https://go.microsoft.com/fwlink/?LinkID=113216). + +## INPUTS + +## OUTPUTS + +## NOTES + +## RELATED LINKS diff --git a/eng/common/TestResources/SubConfig-Helpers.ps1 b/eng/common/TestResources/SubConfig-Helpers.ps1 new file mode 100644 index 00000000..9b69c516 --- /dev/null +++ b/eng/common/TestResources/SubConfig-Helpers.ps1 @@ -0,0 +1,161 @@ +function BuildServiceDirectoryPrefix([string]$serviceName) { + $serviceName = $serviceName -replace '[\./\\]', '_' + return $serviceName.ToUpperInvariant() + "_" +} + +# If the ServiceDirectory has multiple segments use the last directory name +# e.g. D:\foo\bar -> bar or foo/bar -> bar +function GetServiceLeafDirectoryName([string]$serviceDirectory) { + return $serviceDirectory ? (Split-Path -Leaf $serviceDirectory) : "" +} + +function GetUserName() { + $UserName = $env:USER ?? $env:USERNAME + # Remove spaces, etc. that may be in $UserName + $UserName = $UserName -replace '\W' + return $UserName +} + +function GetBaseAndResourceGroupNames( + [string]$baseNameDefault, + [string]$resourceGroupNameDefault, + [string]$user, + [string]$serviceDirectoryName, + [bool]$CI +) { + if ($baseNameDefault) { + $base = $baseNameDefault.ToLowerInvariant() + $group = $resourceGroupNameDefault ? $resourceGroupNameDefault : ("rg-$baseNameDefault".ToLowerInvariant()) + return $base, $group + } + + if ($CI) { + $base = 't' + (New-Guid).ToString('n').Substring(0, 16) + # Format the resource group name based on resource group naming recommendations and limitations. + $generatedGroup = "rg-{0}-$base" -f ($serviceName -replace '[\.\\\/:]', '-'). + Substring(0, [Math]::Min($serviceDirectoryName.Length, 90 - $base.Length - 4)). + Trim('-'). + ToLowerInvariant() + $group = $resourceGroupNameDefault ? $resourceGroupNameDefault : $generatedGroup + + Log "Generated resource base name '$base' and resource group name '$group' for CI build" + + return $base, $group + } + + # Handle service directories in nested directories, e.g. `data/aztables` + $serviceDirectorySafeName = $serviceDirectoryName -replace '[\./\\]', '' + # Seed off resource group name if set to avoid name conflicts with deployments where it is not set + $seed = $resourceGroupNameDefault ? $resourceGroupNameDefault : "${user}${serviceDirectorySafeName}" + $baseNameStream = [IO.MemoryStream]::new([Text.Encoding]::UTF8.GetBytes("$seed")) + # Hash to keep resource names short enough to not break naming restrictions (e.g. keyvault name length) + $base = 't' + (Get-FileHash -InputStream $baseNameStream -Algorithm SHA256).Hash.Substring(0, 16).ToLowerInvariant() + $group = $resourceGroupNameDefault ? $resourceGroupNameDefault : "rg-${user}${serviceDirectorySafeName}".ToLowerInvariant(); + + Log "BaseName was not set. Generating resource group name '$group' and resource base name '$base'" + + return $base, $group +} + +function ShouldMarkValueAsSecret([string]$serviceName, [string]$key, [string]$value, [array]$allowedValues = @()) +{ + $logOutputNonSecret = @( + # Environment Variables + "RESOURCEGROUP_NAME", + # Deployment Outputs + "CLIENT_ID", + "TENANT_ID", + "SUBSCRIPTION_ID", + "RESOURCE_GROUP", + "LOCATION", + "ENVIRONMENT", + "AUTHORITY_HOST", + "RESOURCE_MANAGER_URL", + "SERVICE_MANAGEMENT_URL", + "ENDPOINT_SUFFIX", + "SERVICE_DIRECTORY", + # This is used in many places and is harder to extract from the base subscription config, so hardcode it for now. + "STORAGE_ENDPOINT_SUFFIX", + # Parameters + "Environment", + "SubscriptionId", + "TenantId", + "TestApplicationId", + "TestApplicationOid", + "ProvisionerApplicationId" + ) + + $serviceDirectoryPrefix = BuildServiceDirectoryPrefix $serviceName + + $suffix1 = $key -replace $serviceDirectoryPrefix, "" + $suffix2 = $key -replace "AZURE_", "" + $variants = @($key, $suffix1, $suffix2) + if ($variants | Where-Object { $logOutputNonSecret -contains $_ }) { + return $false + } + + if ($allowedValues -contains $value) { + return $false + } + + return $true +} + +function SetSubscriptionConfiguration([object]$subscriptionConfiguration) +{ + foreach($pair in $subscriptionConfiguration.GetEnumerator()) { + if ($pair.Value -is [Hashtable]) { + foreach($nestedPair in $pair.Value.GetEnumerator()) { + # Mark values as secret so we don't print json blobs containing secrets in the logs. + # Prepend underscore to the variable name, so we can still access the variable names via environment + # variables if they get set subsequently. + if ([Environment]::GetEnvironmentVariable($nestedPair.Name)) { + throw "Environment variable '$($nestedPair.Name)' is already set. Check the tests.yml/ci.yml EnvVars parameter does not conflict with the subscription config json" + } + if (ShouldMarkValueAsSecret "AZURE_" $nestedPair.Name $nestedPair.Value) { + Write-Host "##vso[task.setvariable variable=_$($nestedPair.Name);issecret=true;]$($nestedPair.Value)" + } + } + } else { + if ([Environment]::GetEnvironmentVariable($pair.Name)) { + throw "Environment variable '$($pair.Name)' is already set. Check the tests.yml/ci.yml EnvVars parameter does not conflict with the subscription config json" + } + if (ShouldMarkValueAsSecret "AZURE_" $pair.Name $pair.Value) { + Write-Host "##vso[task.setvariable variable=_$($pair.Name);issecret=true;]$($pair.Value)" + } + } + } + + Write-Host ($subscriptionConfiguration | ConvertTo-Json) + $serialized = $subscriptionConfiguration | ConvertTo-Json -Compress + Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized" +} + +function UpdateSubscriptionConfiguration([object]$subscriptionConfigurationBase, [object]$subscriptionConfiguration) +{ + foreach ($pair in $subscriptionConfiguration.GetEnumerator()) { + if ($pair.Value -is [Hashtable]) { + if (!$subscriptionConfigurationBase.ContainsKey($pair.Name)) { + $subscriptionConfigurationBase[$pair.Name] = @{} + } + foreach($nestedPair in $pair.Value.GetEnumerator()) { + # Mark values as secret so we don't print json blobs containing secrets in the logs. + # Prepend underscore to the variable name, so we can still access the variable names via environment + # variables if they get set subsequently. + if (ShouldMarkValueAsSecret "AZURE_" $nestedPair.Name $nestedPair.Value) { + Write-Host "##vso[task.setvariable variable=_$($nestedPair.Name);issecret=true;]$($nestedPair.Value)" + } + $subscriptionConfigurationBase[$pair.Name][$nestedPair.Name] = $nestedPair.Value + } + } else { + if (ShouldMarkValueAsSecret "AZURE_" $pair.Name $pair.Value) { + Write-Host "##vso[task.setvariable variable=_$($pair.Name);issecret=true;]$($pair.Value)" + } + $subscriptionConfigurationBase[$pair.Name] = $pair.Value + } + } + + $serialized = $subscriptionConfigurationBase | ConvertTo-Json -Compress + Write-Host ($subscriptionConfigurationBase | ConvertTo-Json) + Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]$serialized" +} diff --git a/eng/common/TestResources/Update-TestResources.cmd b/eng/common/TestResources/Update-TestResources.cmd new file mode 100644 index 00000000..94b0c1f2 --- /dev/null +++ b/eng/common/TestResources/Update-TestResources.cmd @@ -0,0 +1,17 @@ +@echo off + +REM Copyright (c) Microsoft Corporation. All rights reserved. +REM Licensed under the MIT License. + +setlocal + +for /f "usebackq delims=" %%i in (`where pwsh 2^>nul`) do ( + set _cmd=%%i +) + +if "%_cmd%"=="" ( + echo Error: PowerShell not found. Please visit https://github.com/powershell/powershell for install instructions. + exit /b 2 +) + +call "%_cmd%" -NoLogo -NoProfile -File "%~dpn0.ps1" %* diff --git a/eng/common/TestResources/Update-TestResources.ps1 b/eng/common/TestResources/Update-TestResources.ps1 new file mode 100644 index 00000000..9592e8a8 --- /dev/null +++ b/eng/common/TestResources/Update-TestResources.ps1 @@ -0,0 +1,204 @@ +#!/usr/bin/env pwsh + +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +#Requires -Version 6.0 +#Requires -PSEdition Core +#Requires -Modules @{ModuleName='Az.Accounts'; ModuleVersion='1.6.4'} +#Requires -Modules @{ModuleName='Az.Resources'; ModuleVersion='1.8.0'} + +[CmdletBinding(DefaultParameterSetName = 'Default')] +param ( + [Parameter(ParameterSetName = 'Default', Position = 0)] + [string] $ServiceDirectory, + + [Parameter(ParameterSetName = 'Default')] + [ValidatePattern('^[-a-zA-Z0-9\.\(\)_]{0,80}(?<=[a-zA-Z0-9\(\)])$')] + [string] $BaseName, + + [Parameter(ParameterSetName = 'ResourceGroup')] + [ValidatePattern('^[-\w\._\(\)]+$')] + [string] $ResourceGroupName, + + [Parameter()] + [ValidatePattern('^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$')] + [string] $SubscriptionId, + + [Parameter()] + [ValidateRange(1, 30*24)] + [int] $DeleteAfterHours = 48 +) + +. $PSScriptRoot/SubConfig-Helpers.ps1 + +# By default stop for any error. +if (!$PSBoundParameters.ContainsKey('ErrorAction')) { + $ErrorActionPreference = 'Stop' +} + +function Log($Message) { + Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message) +} + +function Retry([scriptblock] $Action, [int] $Attempts = 5) { + $attempt = 0 + $sleep = 5 + + while ($attempt -lt $Attempts) { + try { + $attempt++ + return $Action.Invoke() + } catch { + if ($attempt -lt $Attempts) { + $sleep *= 2 + + Write-Warning "Attempt $attempt failed: $_. Trying again in $sleep seconds..." + Start-Sleep -Seconds $sleep + } else { + Write-Error -ErrorRecord $_ + } + } + } +} + +# Support actions to invoke on exit. +$exitActions = @({ + if ($exitActions.Count -gt 1) { + Write-Verbose 'Running registered exit actions' + } +}) + +$serviceName = GetServiceLeafDirectoryName $ServiceDirectory +$BaseName, $ResourceGroupName = GetBaseAndResourceGroupNames ` + -baseNameDefault $BaseName ` + -resourceGroupNameDefault $ResourceGroupName ` + -user (GetUserName) ` + -serviceDirectoryName $serviceName ` + -CI $false + +# This script is intended for interactive users. Make sure they are logged in or fail. +$context = Get-AzContext +if (!$context) { + throw "You must be already logged in to use this script. Run 'Connect-AzAccount' and try again." +} + +# If no subscription was specified, try to select the Azure SDK Developer Playground subscription. +# Ignore errors to leave the automatically selected subscription. +if ($SubscriptionId) { + $currentSubcriptionId = $context.Subscription.Id + if ($currentSubcriptionId -ne $SubscriptionId) { + Log "Selecting subscription '$SubscriptionId'" + $null = Select-AzSubscription -Subscription $SubscriptionId + + $exitActions += { + Log "Selecting previous subscription '$currentSubcriptionId'" + $null = Select-AzSubscription -Subscription $currentSubcriptionId + } + + # Update the context. + $context = Get-AzContext + } +} else { + Log "Attempting to select subscription 'Azure SDK Developer Playground (faa080af-c1d8-40ad-9cce-e1a450ca5b57)'" + $null = Select-AzSubscription -Subscription 'faa080af-c1d8-40ad-9cce-e1a450ca5b57' -ErrorAction Ignore + + # Update the context. + $context = Get-AzContext + + $SubscriptionId = $context.Subscription.Id + $PSBoundParameters['SubscriptionId'] = $SubscriptionId +} + +# Use cache of well-known team subs without having to be authenticated. +$wellKnownSubscriptions = @{ + 'faa080af-c1d8-40ad-9cce-e1a450ca5b57' = 'Azure SDK Developer Playground' + 'a18897a6-7e44-457d-9260-f2854c0aca42' = 'Azure SDK Engineering System' + '2cd617ea-1866-46b1-90e3-fffb087ebf9b' = 'Azure SDK Test Resources' +} + +# Print which subscription is currently selected. +$subscriptionName = $context.Subscription.Id +if ($wellKnownSubscriptions.ContainsKey($subscriptionName)) { + $subscriptionName = '{0} ({1})' -f $wellKnownSubscriptions[$subscriptionName], $subscriptionName +} + +Log "Selected subscription '$subscriptionName'" + +# try..finally will also trap Ctrl+C. +try { + Log "Getting resource group '$ResourceGroupName'" + + $resourceGroup = Get-AzResourceGroup -Name $ResourceGroupName + + # Update DeleteAfter + $deleteAfter = [DateTime]::UtcNow.AddHours($DeleteAfterHours).ToString('o') + + Log "Updating DeleteAfter to '$deleteAfter'" + Write-Warning "Any clean-up scripts running against subscription '$SubscriptionId' may delete resource group '$ResourceGroupName' after $DeleteAfterHours hours." + if (!$resourceGroup.Tags) { + $resourceGroup.Tags = @{} + } + $resourceGroup.Tags['DeleteAfter'] = $deleteAfter + + Log "Updating resource group '$ResourceGroupName'" + Retry { + # Allow the resource group to write to output. + Set-AzResourceGroup -Name $ResourceGroupName -Tag $resourceGroup.Tags + } +} finally { + $exitActions.Invoke() +} + +<# +.SYNOPSIS +Updates a resource group previously deployed for a service directory. + +.DESCRIPTION +Updates a resource group that was created using New-TestResources.ps1. +You can use this, for example, to update the `DeleteAfterHours` property +to keep an existing resource group deployed for a longer period of time. + +.PARAMETER ServiceDirectory +A directory under 'sdk' in the repository root - optionally with subdirectories +specified - in which to discover ARM templates named 'test-resources.json'. +This can also be an absolute path or specify parent directories. + +.PARAMETER BaseName +A name to use in the resource group and passed to the ARM template as 'baseName'. +This will update the resource group named 'rg-' + +.PARAMETER ResourceGroupName +The name of the resource group to update. + +.PARAMETER SubscriptionId +Optional subscription ID to use when deleting resources when logging in as a +provisioner. You can also use Set-AzContext if not provisioning. + +If you do not specify a SubscriptionId and are not logged in, one will be +automatically selected for you by the Connect-AzAccount cmdlet. + +Once you are logged in (or were previously), the selected SubscriptionId +will be used for subsequent operations that are specific to a subscription. + +.PARAMETER DeleteAfterHours +Positive integer number of hours from the current time to set the +'DeleteAfter' tag on the created resource group. The computed value is a +timestamp of the form "2020-03-04T09:07:04.3083910Z". + +An optional cleanup process can delete resource groups whose "DeleteAfter" +timestamp is less than the current time. + +.EXAMPLE +Update-TestResources.ps1 keyvault -DeleteAfterHours 24 + +Update the 'rg-${USERNAME}keyvault` resource group to be deleted after 24 +hours from now if a clean-up script is running against the current subscription. + +.EXAMPLE +Update-TestResources.ps1 -ResourceGroupName rg-userkeyvault -Subscription fa9c6912-f641-4226-806c-5139584b89ca + +Update the 'rg-userkeyvault' resource group to be deleted after 48 +hours from now if a clean-up script is running against the subscription 'fa9c6912-f641-4226-806c-5139584b89ca'. + +#> diff --git a/eng/common/TestResources/Update-TestResources.ps1.md b/eng/common/TestResources/Update-TestResources.ps1.md new file mode 100644 index 00000000..17d31511 --- /dev/null +++ b/eng/common/TestResources/Update-TestResources.ps1.md @@ -0,0 +1,153 @@ +--- +external help file: -help.xml +Module Name: +online version: +schema: 2.0.0 +--- + +# Update-TestResources.ps1 + +## SYNOPSIS +Updates a resource group previously deployed for a service directory. + +## SYNTAX + +### Default (Default) +``` +Update-TestResources.ps1 [-ServiceDirectory] [-BaseName ] [-SubscriptionId ] + [-DeleteAfterHours ] [] +``` + +### ResourceGroup +``` +Update-TestResources.ps1 [-ResourceGroupName ] [-SubscriptionId ] [-DeleteAfterHours ] + [] +``` + +## DESCRIPTION +Updates a resource group that was created using New-TestResources.ps1. +You can use this, for example, to update the \`DeleteAfterHours\` property +to keep an existing resource group deployed for a longer period of time. + +## EXAMPLES + +### EXAMPLE 1 +``` +Update-TestResources.ps1 keyvault -DeleteAfterHours 24 +``` + +Update the 'rg-${USERNAME}keyvault\` resource group to be deleted after 24 +hours from now if a clean-up script is running against the current subscription. + +### EXAMPLE 2 +``` +Update-TestResources.ps1 -ResourceGroupName rg-userkeyvault -Subscription fa9c6912-f641-4226-806c-5139584b89ca +``` + +Update the 'rg-userkeyvault' resource group to be deleted after 48 +hours from now if a clean-up script is running against the subscription 'fa9c6912-f641-4226-806c-5139584b89ca'. + +## PARAMETERS + +### -ServiceDirectory +A directory under 'sdk' in the repository root - optionally with subdirectories +specified - in which to discover ARM templates named 'test-resources.json'. +This can also be an absolute path or specify parent directories. + +```yaml +Type: String +Parameter Sets: Default +Aliases: + +Required: True +Position: 1 +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -BaseName +A name to use in the resource group and passed to the ARM template as 'baseName'. +This will update the resource group named 'rg-\' + +```yaml +Type: String +Parameter Sets: Default +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ResourceGroupName +The name of the resource group to update. + +```yaml +Type: String +Parameter Sets: ResourceGroup +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -SubscriptionId +Optional subscription ID to use when deleting resources when logging in as a +provisioner. +You can also use Set-AzContext if not provisioning. + +If you do not specify a SubscriptionId and are not logged in, one will be +automatically selected for you by the Connect-AzAccount cmdlet. + +Once you are logged in (or were previously), the selected SubscriptionId +will be used for subsequent operations that are specific to a subscription. + +```yaml +Type: String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -DeleteAfterHours +Positive integer number of hours from the current time to set the +'DeleteAfter' tag on the created resource group. +The computed value is a +timestamp of the form "2020-03-04T09:07:04.3083910Z". + +An optional cleanup process can delete resource groups whose "DeleteAfter" +timestamp is less than the current time. + +```yaml +Type: Int32 +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: 48 +Accept pipeline input: False +Accept wildcard characters: False +``` + +### CommonParameters +This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see [about_CommonParameters](https://go.microsoft.com/fwlink/?LinkID=113216). + +## INPUTS + +## OUTPUTS + +## NOTES + +## RELATED LINKS diff --git a/eng/common/TestResources/build-test-resource-config.yml b/eng/common/TestResources/build-test-resource-config.yml new file mode 100644 index 00000000..2f158c47 --- /dev/null +++ b/eng/common/TestResources/build-test-resource-config.yml @@ -0,0 +1,46 @@ +parameters: + - name: SubscriptionConfiguration + type: string + default: $(sub-config-azure-cloud-test-resources) + - name: SubscriptionConfigurations + type: object + default: null + # EnvVars is used to help diagnose variable conflict issues early + - name: EnvVars + type: object + default: null + +steps: + - ${{ if parameters.SubscriptionConfiguration }}: + - pwsh: | + $config = @' + ${{ parameters.SubscriptionConfiguration }} + '@ | ConvertFrom-Json -AsHashtable + + . ./eng/common/TestResources/SubConfig-Helpers.ps1 + SetSubscriptionConfiguration $config + displayName: Initialize SubscriptionConfiguration variable + ${{ if parameters.EnvVars }}: + env: ${{ parameters.EnvVars }} + + - ${{ if parameters.SubscriptionConfigurations }}: + - pwsh: | + Write-Host "##vso[task.setvariable variable=SubscriptionConfiguration;]{}" + displayName: Initialize SubscriptionConfiguration variable for merging + condition: eq(variables['SubscriptionConfiguration'], '') + + - ${{ each config in parameters.SubscriptionConfigurations }}: + - pwsh: | + $configBase = @' + $(SubscriptionConfiguration) + '@ | ConvertFrom-Json -AsHashtable + $config = @' + ${{ config }} + '@ | ConvertFrom-Json -AsHashtable + + . ./eng/common/TestResources/SubConfig-Helpers.ps1 + UpdateSubscriptionConfiguration $configBase $config + + displayName: Merge Test Resource Configurations + ${{ if parameters.EnvVars }}: + env: ${{ parameters.EnvVars }} diff --git a/eng/common/TestResources/clouds/AzureChinaCloud.json b/eng/common/TestResources/clouds/AzureChinaCloud.json new file mode 100644 index 00000000..8c1ebd28 --- /dev/null +++ b/eng/common/TestResources/clouds/AzureChinaCloud.json @@ -0,0 +1,15 @@ +{ + "azConfigEndpointSuffix": ".azconfig.azure.cn", + "azureAuthorityHost": "https://login.chinacloudapi.cn/", + "cognitiveServicesEndpointSuffix": ".cognitiveservices.azure.cn", + "containerRegistryEndpointSuffix": ".azurecr.cn", + "cosmosEndpointSuffix": "cosmos.azure.cn", + "enableStorageVersioning": false, + "keyVaultDomainSuffix": ".vault.azure.cn", + "keyVaultEndpointSuffix": ".vault.azure.cn", + "keyVaultSku": "standard", + "searchEndpointSuffix": "search.azure.cn", + "serviceBusEndpointSuffix": ".servicebus.chinacloudapi.cn", + "storageEndpointSuffix": "core.chinacloudapi.cn", + "textAnalyticsSku": "S" +} diff --git a/eng/common/TestResources/clouds/AzureCloud.json b/eng/common/TestResources/clouds/AzureCloud.json new file mode 100644 index 00000000..97ab7954 --- /dev/null +++ b/eng/common/TestResources/clouds/AzureCloud.json @@ -0,0 +1,9 @@ +{ + "azureAuthorityHost": "https://login.microsoftonline.com/", + "cognitiveServicesEndpointSuffix": ".cognitiveservices.azure.com", + "communicationServicesEndpointSuffix": ".communication.azure.com", + "cosmosEndpointSuffix": "cosmos.azure.com", + "keyVaultDomainSuffix": ".vault.azure.net", + "keyVaultEndpointSuffix": ".vault.azure.net", + "storageEndpointSuffix": "core.windows.net" +} diff --git a/eng/common/TestResources/clouds/AzureUSGovernment.json b/eng/common/TestResources/clouds/AzureUSGovernment.json new file mode 100644 index 00000000..989975aa --- /dev/null +++ b/eng/common/TestResources/clouds/AzureUSGovernment.json @@ -0,0 +1,15 @@ +{ + "azConfigEndpointSuffix": ".azconfig.azure.us", + "azureAuthorityHost": "https://login.microsoftonline.us/", + "cognitiveServicesEndpointSuffix": ".cognitiveservices.azure.us", + "containerRegistryEndpointSuffix": ".azurecr.us", + "cosmosEndpointSuffix": "cosmos.azure.us", + "enableStorageVersioning": false, + "formRecognizerLocation": "usgovvirginia", + "keyVaultDomainSuffix": ".vault.usgovcloudapi.net", + "keyVaultEndpointSuffix": ".vault.usgovcloudapi.net", + "keyVaultSku": "premium", + "searchEndpointSuffix": "search.azure.us", + "serviceBusEndpointSuffix": ".servicebus.usgovcloudapi.net", + "storageEndpointSuffix": "core.usgovcloudapi.net" +} diff --git a/eng/common/TestResources/deploy-test-resources.yml b/eng/common/TestResources/deploy-test-resources.yml new file mode 100644 index 00000000..a0c68f33 --- /dev/null +++ b/eng/common/TestResources/deploy-test-resources.yml @@ -0,0 +1,63 @@ +parameters: + ServiceDirectory: not-set + ArmTemplateParameters: '@{}' + DeleteAfterHours: 8 + Location: '' + SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) + ResourceType: test + +# SubscriptionConfiguration will be splatted into the parameters of the test +# resources script. It should be JSON in the form: +# { +# "SubscriptionId": "", +# "TenantId": "", +# "TestApplicationId": "", +# "TestApplicationSecret": "", +# "ProvisionerApplicationId": "", +# "ProvisionerApplicationSecret": "", +# "Environment": "AzureCloud | AzureGov | AzureChina | " +# "EnvironmentVariables": { +# "SERVICE_MANAGEMENT_URL": "", +# "STORAGE_ENDPOINT_SUFFIX": "", +# "RESOURCE_MANAGER_URL": "", +# "SEARCH_ENDPOINT_SUFFIX": "", +# "COSMOS_TABLES_ENDPOINT_SUFFIX": "" +# }, +# "ArmTemplateParameters": { +# "keyVaultDomainSuffix": "", +# "storageEndpointSuffix": "", +# "endpointSuffix": "", +# "azureAuthorityHost": "", +# "keyVaultEndpointSuffix": "" +# } +# } + + +steps: + - template: /eng/common/pipelines/templates/steps/cache-ps-modules.yml + + - template: /eng/common/TestResources/setup-environments.yml + + - pwsh: | + eng/common/scripts/Import-AzModules.ps1 + + $subscriptionConfiguration = @' + ${{ parameters.SubscriptionConfiguration }} + '@ | ConvertFrom-Json -AsHashtable; + + # The subscriptionConfiguration may have ArmTemplateParameters defined, so + # pass those in via the ArmTemplateParameters flag, and handle any + # additional parameters from the pipelines via AdditionalParameters + eng/common/TestResources/New-TestResources.ps1 ` + -ResourceType '${{ parameters.ResourceType }}' ` + -ServiceDirectory '${{ parameters.ServiceDirectory }}' ` + -Location '${{ parameters.Location }}' ` + -DeleteAfterHours '${{ parameters.DeleteAfterHours }}' ` + @subscriptionConfiguration ` + -AdditionalParameters ${{ parameters.ArmTemplateParameters }} ` + -CI ` + -Force ` + -Verbose | Out-Null + displayName: Deploy test resources + env: + TEMP: $(Agent.TempDirectory) diff --git a/eng/common/TestResources/remove-test-resources.yml b/eng/common/TestResources/remove-test-resources.yml new file mode 100644 index 00000000..9675f58e --- /dev/null +++ b/eng/common/TestResources/remove-test-resources.yml @@ -0,0 +1,40 @@ +# Assumes steps in deploy-test-resources.yml was run previously. Requires +# environment variable: _RESOURCE_GROUP and Az PowerShell module + +parameters: + ServiceDirectory: '' + SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) + ResourceType: test + +# SubscriptionConfiguration will be splat into the parameters of the test +# resources script. It should be JSON in the form: +# { +# "SubscriptionId": "", +# "TenantId": "", +# "TestApplicationId": "", +# "TestApplicationSecret": "", +# "ProvisionerApplicationId": "", +# "ProvisionerApplicationSecret": "", +# "Environment": "AzureCloud | AzureGov | AzureChina | " +# } +# The Remove-TestResources.ps1 script accommodates extra parameters so it will +# not error when parameters are provided which the script doesn't use. + +steps: + - pwsh: | + eng/common/scripts/Import-AzModules.ps1 + + $subscriptionConfiguration = @" + ${{ parameters.SubscriptionConfiguration }} + "@ | ConvertFrom-Json -AsHashtable; + + eng/common/TestResources/Remove-TestResources.ps1 ` + @subscriptionConfiguration ` + -ResourceType '${{ parameters.ResourceType }}' ` + -ServiceDirectory "${{ parameters.ServiceDirectory }}" ` + -CI ` + -Force ` + -Verbose + displayName: Remove test resources + condition: and(eq(variables['CI_HAS_DEPLOYED_RESOURCES'], 'true'), ne(variables['Skip.RemoveTestResources'], 'true')) + continueOnError: true diff --git a/eng/common/TestResources/setup-environments.yml b/eng/common/TestResources/setup-environments.yml new file mode 100644 index 00000000..a829b8da --- /dev/null +++ b/eng/common/TestResources/setup-environments.yml @@ -0,0 +1,34 @@ +# Cloud Configuration will be splat into parameters of `Add-AzEnvironment`. It +# should be JSON in the form (not all fields are required): +# { +# "Name": "", +# "PublishSettingsFileUrl": "", +# "ServiceEndpoint": "", +# "ManagementPortalUrl": "", +# "ActiveDirectoryEndpoint": "", +# "ActiveDirectoryServiceEndpointResourceId": "", +# "ResourceManagerEndpoint": "", +# "GalleryEndpoint": "", +# "GraphEndpoint": "", +# "GraphAudience": "", +# "AzureKeyVaultDnsSuffix": "", +# "AzureKeyVaultServiceEndpointResourceId": "" +# } + +steps: + - bash: sudo chown -R runner ~/.Azure + displayName: (MacOS) Grant access to ~/.Azure + condition: contains(variables['OSVmImage'], 'mac') + + - task: Powershell@2 + displayName: Setup Az Modules and Dogfood Environment + inputs: + targetType: inline + pwsh: true + script: | + eng/common/scripts/Import-AzModules.ps1 + + $environmentSpec = @" + $(env-config-dogfood) + "@ | ConvertFrom-Json -AsHashtable; + Add-AzEnvironment @environmentSpec diff --git a/eng/common/docgeneration/Generate-DocIndex.ps1 b/eng/common/docgeneration/Generate-DocIndex.ps1 new file mode 100644 index 00000000..2b8d5946 --- /dev/null +++ b/eng/common/docgeneration/Generate-DocIndex.ps1 @@ -0,0 +1,189 @@ +# Generates an index page for cataloging different versions of the Docs +[CmdletBinding()] +Param ( + $DocFx, + $RepoRoot, + $DocGenDir, + $DocOutDir = "${RepoRoot}/docfx_project", + $DocfxJsonPath = "${PSScriptRoot}\docfx.json", + $MainJsPath = "${PSScriptRoot}\templates\matthews\styles\main.js" +) +. "${PSScriptRoot}\..\scripts\common.ps1" + +# Fetch a list of "artifacts" from blob storage corresponding to the given +# language (-storagePrefix). Remove the prefix from the path names to arrive at +# an "artifact" name. +function Get-BlobStorage-Artifacts( + $blobDirectoryRegex, + $blobArtifactsReplacement, + $storageAccountName, + $storageContainerName, + $storagePrefix +) { + LogDebug "Reading artifact from storage blob ..." + # "--only-show-errors" suppresses warnings about the fact that the az CLI is not authenticated + # "--query '[].name'" returns a list of only blob names + # "--num-results *" handles pagination so the caller does not have to + $artifacts = az storage blob list ` + --account-name $storageAccountName ` + --container-name $storageContainerName ` + --prefix $storagePrefix ` + --delimiter / ` + --only-show-errors ` + --query '[].name' ` + --num-results * | ConvertFrom-Json + LogDebug "Number of artifacts found: $($artifacts.Length)" + + # example: "python/azure-storage-blob" -> "azure-storage-blob" + $artifacts = $artifacts.ForEach({ $_ -replace $blobDirectoryRegex, $blobArtifactsReplacement }) + return $artifacts +} + +function Get-TocMapping { + Param ( + [Parameter(Mandatory = $true)] [Object[]] $metadata, + [Parameter(Mandatory = $true)] [String[]] $artifacts + ) + # Used for sorting the toc display order + $orderServiceMapping = @{} + + foreach ($artifact in $artifacts) { + $packageInfo = $metadata | ? { $_.Package -eq $artifact -and $_.Hide -ne "true" } + + $serviceName = "" + if (!$packageInfo) { + LogDebug "There is no service name for artifact $artifact or it is marked as hidden. Please check csv of Azure/azure-sdk/_data/release/latest repo if this is intended. " + continue + } + elseif (!$packageInfo[0].ServiceName) { + LogWarning "There is no service name for artifact $artifact. Please check csv of Azure/azure-sdk/_data/release/latest repo if this is intended. " + # If no service name retrieved, print out warning message, and put it into Other page. + $serviceName = "Other" + } + else { + if ($packageInfo.Length -gt 1) { + LogWarning "There are more than 1 packages fetched out for artifact $artifact. Please check csv of Azure/azure-sdk/_data/release/latest repo if this is intended. " + } + $serviceName = $packageInfo[0].ServiceName.Trim() + } + + # Define the order of "New", "Type", if not match, return the length of the array. + $CustomOrder_New = "true", "false", "" + $newIndex = $CustomOrder_New.IndexOf($packageInfo[0].New.ToLower()) + $newIndex = $newIndex -eq -1 ? $CustomOrder_New.Count : $newIndex + $CustomOrder_Type = "client", "mgmt", "compat", "spring", "" + $typeIndex = $CustomOrder_Type.IndexOf($packageInfo[0].Type.ToLower()) + $typeIndex = $typeIndex -eq -1 ? $CustomOrder_Type.Count : $typeIndex + $orderServiceMapping[$artifact] = [PSCustomObject][ordered]@{ + NewIndex = $newIndex + TypeIndex = $typeIndex + ServiceName = $serviceName + DisplayName = $packageInfo[0].DisplayName.Trim() + Artifact = $artifact + } + } + return $orderServiceMapping +} + +function GenerateDocfxTocContent([Hashtable]$tocContent, [String]$lang, [String]$campaignId = "UA-62780441-46") { + LogDebug "Start generating the docfx toc and build docfx site..." + + LogDebug "Initializing Default DocFx Site..." + & $($DocFx) init -q -o "${DocOutDir}" + # The line below is used for testing in local + #docfx init -q -o "${DocOutDir}" + LogDebug "Copying template and configuration..." + New-Item -Path "${DocOutDir}" -Name "templates" -ItemType "directory" -Force + Copy-Item "${DocGenDir}/templates/*" -Destination "${DocOutDir}/templates" -Force -Recurse + + $headerTemplateLocation = "${DocOutDir}/templates/matthews/partials/head.tmpl.partial" + + if ($campaignId -and (Test-Path $headerTemplateLocation)){ + $headerTemplateContent = Get-Content -Path $headerTemplateLocation -Raw + $headerTemplateContent = $headerTemplateContent -replace "GA_CAMPAIGN_ID", $campaignId + Set-Content -Path $headerTemplateLocation -Value $headerTemplateContent -NoNewline + } + + Copy-Item "${DocGenDir}/docfx.json" -Destination "${DocOutDir}/" -Force + $YmlPath = "${DocOutDir}/api" + New-Item -Path $YmlPath -Name "toc.yml" -Force + $visitedService = @{} + # Sort and display toc service name by alphabetical order, and then sort artifact by order. + $sortedToc = $tocContent.Values | Sort-Object ServiceName, NewIndex, TypeIndex, DisplayName, Artifact + foreach ($serviceMapping in $sortedToc) { + $artifact = $serviceMapping.Artifact + $serviceName = $serviceMapping.ServiceName + $displayName = $serviceMapping.DisplayName + + # handle spaces in service name, EG "Confidential Ledger" + # handle / in service name, EG "Database for MySQL/PostgreSQL". Leaving a "/" present will generate a bad link location. + $fileName = ($serviceName -replace '\s', '').Replace("/","").ToLower().Trim() + if ($visitedService.ContainsKey($serviceName)) { + if ($displayName) { + Add-Content -Path "$($YmlPath)/${fileName}.md" -Value "#### $artifact`n##### ($displayName)" + } + else { + Add-Content -Path "$($YmlPath)/${fileName}.md" -Value "#### $artifact" + } + } + else { + Add-Content -Path "$($YmlPath)/toc.yml" -Value "- name: ${serviceName}`r`n href: ${fileName}.md" + New-Item -Path $YmlPath -Name "${fileName}.md" -Force + if ($displayName) { + Add-Content -Path "$($YmlPath)/${fileName}.md" -Value "#### $artifact`n##### ($displayName)" + } + else { + Add-Content -Path "$($YmlPath)/${fileName}.md" -Value "#### $artifact" + } + $visitedService[$serviceName] = $true + } + } + + # Generate toc homepage. + LogDebug "Creating Site Title and Navigation..." + New-Item -Path "${DocOutDir}" -Name "toc.yml" -Force + Add-Content -Path "${DocOutDir}/toc.yml" -Value "- name: Azure SDK for $lang APIs`r`n href: api/`r`n homepage: api/index.md" + + LogDebug "Copying root markdowns" + Copy-Item "$($RepoRoot)/README.md" -Destination "${DocOutDir}/api/index.md" -Force + Copy-Item "$($RepoRoot)/CONTRIBUTING.md" -Destination "${DocOutDir}/api/CONTRIBUTING.md" -Force + + LogDebug "Building site..." + & $($DocFx) build "${DocOutDir}/docfx.json" + # The line below is used for testing in local + #docfx build "${DocOutDir}/docfx.json" + Copy-Item "${DocGenDir}/assets/logo.svg" -Destination "${DocOutDir}/_site/" -Force +} + +function UpdateDocIndexFiles { + Param ( + [Parameter(Mandatory=$false)] [String]$appTitleLang = $Language, + [Parameter(Mandatory=$false)] [String]$lang = $Language, + [Parameter(Mandatory=$false)] [String]$packageRegex = "`"`"", + [Parameter(Mandatory=$false)] [String]$regexReplacement = "" + ) + # Update docfx.json + $docfxContent = Get-Content -Path $DocfxJsonPath -Raw + $docfxContent = $docfxContent -replace "`"_appTitle`": `"`"", "`"_appTitle`": `"Azure SDK for $appTitleLang`"" + $docfxContent = $docfxContent -replace "`"_appFooter`": `"`"", "`"_appFooter`": `"Azure SDK for $appTitleLang`"" + Set-Content -Path $DocfxJsonPath -Value $docfxContent -NoNewline + # Update main.js var lang + $mainJsContent = Get-Content -Path $MainJsPath -Raw + $mainJsContent = $mainJsContent -replace "var SELECTED_LANGUAGE = ''", "var SELECTED_LANGUAGE = '$lang'" + # Update main.js package regex and replacement + $mainJsContent = $mainJsContent -replace "var PACKAGE_REGEX = ''", "var PACKAGE_REGEX = $packageRegex" + $mainJsContent = $mainJsContent -replace "var PACKAGE_REPLACEMENT = ''", "var PACKAGE_REPLACEMENT = `"$regexReplacement`"" + + Set-Content -Path $MainJsPath -Value $mainJsContent -NoNewline +} + +if ($GetGithubIoDocIndexFn -and (Test-Path "function:$GetGithubIoDocIndexFn")) +{ + &$GetGithubIoDocIndexFn +} +else +{ + LogWarning "The function for 'GetGithubIoDocIndexFn' was not found.` + Make sure it is present in eng/scripts/Language-Settings.ps1 and referenced in eng/common/scripts/common.ps1.` + See https://github.com/Azure/azure-sdk-tools/blob/main/doc/common/common_engsys.md#code-structure" +} diff --git a/eng/common/docgeneration/assets/logo.svg b/eng/common/docgeneration/assets/logo.svg new file mode 100644 index 00000000..5da99f40 --- /dev/null +++ b/eng/common/docgeneration/assets/logo.svg @@ -0,0 +1,76 @@ + + + + diff --git a/eng/common/docgeneration/docfx.json b/eng/common/docgeneration/docfx.json new file mode 100644 index 00000000..837478ef --- /dev/null +++ b/eng/common/docgeneration/docfx.json @@ -0,0 +1,72 @@ +{ + "metadata": [ + { + "src": [ + { + "files": [ + "src/**.csproj" + ] + } + ], + "dest": "api", + "disableGitFeatures": false, + "disableDefaultFilter": false + } + ], + "build": { + "content": [ + { + "files": [ + "api/**.yml", + "api/**.md", + "api/index.md" + ] + }, + { + "files": [ + "toc.yml", + "*.md" + ] + } + ], + "resource": [ + { + "files": [ + "images/**" + ] + } + ], + "overwrite": [ + { + "files": [ + "apidoc/**.md" + ], + "exclude": [ + "obj/**", + "_site/**" + ] + } + ], + "dest": "_site", + "globalMetadataFiles": [], + "fileMetadataFiles": [], + "template": [ + "default", + "templates/matthews" + ], + "postProcessors": [], + "markdownEngineName": "markdig", + "noLangKeyword": false, + "keepFileLink": false, + "cleanupCacheHistory": false, + "disableGitFeatures": false, + "globalMetadata": { + "_appTitle": "", + "_appFooter": "", + "_enableSearch": false, + "_enableNewTab": true, + "_appFaviconPath": "https://c.s-microsoft.com/favicon.ico?v2", + "_disableContribution": true + } + } +} diff --git a/eng/common/docgeneration/templates/matthews/partials/affix.tmpl.partial b/eng/common/docgeneration/templates/matthews/partials/affix.tmpl.partial new file mode 100644 index 00000000..43a33d01 --- /dev/null +++ b/eng/common/docgeneration/templates/matthews/partials/affix.tmpl.partial @@ -0,0 +1,17 @@ +{{^_disableContribution}} +
+ {{#docurl}} + + {{/docurl}} + {{#sourceurl}} + + {{/sourceurl}} +
+{{/_disableContribution}} + + diff --git a/eng/common/docgeneration/templates/matthews/partials/class.header.tmpl.partial b/eng/common/docgeneration/templates/matthews/partials/class.header.tmpl.partial new file mode 100644 index 00000000..db73a4bd --- /dev/null +++ b/eng/common/docgeneration/templates/matthews/partials/class.header.tmpl.partial @@ -0,0 +1,100 @@ +

{{>partials/title}}

+
{{{summary}}}
+
{{{conceptual}}}
+ +{{#inClass}} +
+
{{__global.inheritance}}
+ {{#inheritance}} +
{{{specName.0.value}}}
+ {{/inheritance}} +
{{name.0.value}}
+
+{{/inClass}} + +{{#derivedClasses}} +
{{{specName.0.value}}}
+{{/derivedClasses}} + +{{#inheritedMembers.0}} +
+
{{__global.inheritedMembers}}
+{{/inheritedMembers.0}} +{{#inheritedMembers}} +
+ {{#definition}} + + {{/definition}} + {{^definition}} + + {{/definition}} +
+{{/inheritedMembers}} +{{#inheritedMembers.0}} +
+{{/inheritedMembers.0}} + +
{{__global.namespace}}: {{namespace}}
+
{{__global.assembly}}: {{assemblies.0}}.dll
+ +
{{__global.syntax}}
+
+
{{syntax.content.0.value}}
+
+ +{{#syntax.parameters.0}} +
{{__global.parameters}}
+ +{{/syntax.parameters.0}} +{{#syntax.parameters}} + + + +{{/syntax.parameters}} +{{#syntax.parameters.0}} +
+ {{{type.specName.0.value}}} + {{{id}}} +

{{{description}}}

+
+{{/syntax.parameters.0}} + +{{#syntax.return}} +
{{__global.returns}}
+ + + + +
+ {{{type.specName.0.value}}} +

{{{description}}}

+
+{{/syntax.return}} + +{{#syntax.typeParameters.0}} +
{{__global.typeParameters}}
+ +{{/syntax.typeParameters.0}} +{{#syntax.typeParameters}} + + + +{{/syntax.typeParameters}} +{{#syntax.typeParameters.0}} +
+ {{{id}}} +

{{{description}}}

+
+{{/syntax.typeParameters.0}} + +{{#example.0}} +
{{__global.examples}}
+{{/example.0}} +{{#example}} +{{{.}}} +{{/example}} + +{{#remarks}} +
{{__global.remarks}}
+
{{{remarks}}}
+{{/remarks}} \ No newline at end of file diff --git a/eng/common/docgeneration/templates/matthews/partials/class.tmpl.partial b/eng/common/docgeneration/templates/matthews/partials/class.tmpl.partial new file mode 100644 index 00000000..37a69fa1 --- /dev/null +++ b/eng/common/docgeneration/templates/matthews/partials/class.tmpl.partial @@ -0,0 +1,210 @@ +{{>partials/class.header}} +{{#children}} +

{{>partials/classSubtitle}}

+{{#children}} + +{{^_disableContribution}} +{{#docurl}} + + + + +{{/docurl}} +{{#sourceurl}} + + + +{{/sourceurl}} +{{/_disableContribution}} + +{{#overload}} + +{{/overload}} + +

{{name.0.value}}

+ +
+
{{{summary}}}
+
{{{conceptual}}}
+
{{__global.declaration}}
+ +{{#syntax}} +
+
{{syntax.content.0.value}}
+
+ +{{#parameters.0}} +
{{__global.parameters}}
+ +{{/parameters.0}} +{{#parameters}} + + + +{{/parameters}} +{{#parameters.0}} +
+ {{{type.specName.0.value}}} + {{{id}}} +

{{{description}}}

+
+{{/parameters.0}} + +{{#return}} +
{{__global.returns}}
+ + + + +
+ {{{type.specName.0.value}}} +

{{{description}}}

+
+{{/return}} + +{{#typeParameters.0}} +
{{__global.typeParameters}}
+ +{{/typeParameters.0}} +{{#typeParameters}} + + + +{{/typeParameters}} +{{#typeParameters.0}} +
+ {{{id}}} +

{{{description}}}

+
+{{/typeParameters.0}} + +{{#fieldValue}} +
{{__global.fieldValue}}
+ + + + +
+ {{{type.specName.0.value}}} +

{{{description}}}

+
+{{/fieldValue}} + +{{#propertyValue}} +
{{__global.propertyValue}}
+ + + + +
+ {{{type.specName.0.value}}} +

{{{description}}}

+
+{{/propertyValue}} + +{{#eventType}} +
{{__global.eventType}}
+ + + + +
+ {{{type.specName.0.value}}} +

{{{description}}}

+
+{{/eventType}} +{{/syntax}} + +{{#overridden}} +
{{__global.overrides}}
+
+{{/overridden}} + +{{#implements.0}} +
{{__global.implements}}
+{{/implements.0}} +{{#implements}} + {{#definition}} +
+ {{/definition}} + {{^definition}} +
+ {{/definition}} +{{/implements}} + +{{#example.0}} +
{{__global.examples}}
+{{/example.0}} +{{#example}} +{{{.}}} +{{/example}} + +{{#remarks}} +
{{__global.remarks}}
+
{{{remarks}}}
+{{/remarks}} + +{{#exceptions.0}} +
{{__global.exceptions}}
+ +{{/exceptions.0}} +{{#exceptions}} + + + +{{/exceptions}} +{{#exceptions.0}} +
+ {{{type.specName.0.value}}} +

{{{description}}}

+
+{{/exceptions.0}} + +{{#seealso.0}} +
{{__global.seealso}}
+
+{{/seealso.0}} +{{#seealso}} + {{#isCref}} +
{{{type.specName.0.value}}}
+ {{/isCref}} + {{^isCref}} +
{{{url}}}
+ {{/isCref}} +{{/seealso}} +{{#seealso.0}} +
+{{/seealso.0}} +
+{{/children}} +{{/children}} + +{{#extensionMethods.0}} +

{{__global.extensionMethods}}

+{{/extensionMethods.0}} +{{#extensionMethods}} +
+ {{#definition}} + + {{/definition}} + {{^definition}} + + {{/definition}} +
+{{/extensionMethods}} + +{{#seealso.0}} +

{{__global.seealso}}

+
+{{/seealso.0}} +{{#seealso}} + {{#isCref}} +
{{{type.specName.0.value}}}
+ {{/isCref}} + {{^isCref}} +
{{{url}}}
+ {{/isCref}} +{{/seealso}} +{{#seealso.0}} +
+{{/seealso.0}} diff --git a/eng/common/docgeneration/templates/matthews/partials/enum.tmpl.partial b/eng/common/docgeneration/templates/matthews/partials/enum.tmpl.partial new file mode 100644 index 00000000..91e7ede1 --- /dev/null +++ b/eng/common/docgeneration/templates/matthews/partials/enum.tmpl.partial @@ -0,0 +1,24 @@ +{{>partials/class.header}} +{{#children}} + {{#children}} +

{{name.0.value}}

+ +
+

{{{summary}}}

+
+ {{/children}} +{{/children}} + +{{#extensionMethods.0}} +

{{__global.extensionMethods}}

+{{/extensionMethods.0}} +{{#extensionMethods}} +
+ {{#definition}} + + {{/definition}} + {{^definition}} + + {{/definition}} +
+{{/extensionMethods}} diff --git a/eng/common/docgeneration/templates/matthews/partials/head.tmpl.partial b/eng/common/docgeneration/templates/matthews/partials/head.tmpl.partial new file mode 100644 index 00000000..e97616c1 --- /dev/null +++ b/eng/common/docgeneration/templates/matthews/partials/head.tmpl.partial @@ -0,0 +1,28 @@ + + + + {{#title}}{{title}}{{/title}}{{^title}}{{>partials/title}}{{/title}} {{#_appTitle}}| {{_appTitle}} {{/_appTitle}} + + + + {{#_description}}{{/_description}} + + + + + + + {{#_noindex}}{{/_noindex}} + {{#_enableSearch}}{{/_enableSearch}} + {{#_enableNewTab}}{{/_enableNewTab}} + + + + + diff --git a/eng/common/docgeneration/templates/matthews/partials/namespace.tmpl.partial b/eng/common/docgeneration/templates/matthews/partials/namespace.tmpl.partial new file mode 100644 index 00000000..f607a3dc --- /dev/null +++ b/eng/common/docgeneration/templates/matthews/partials/namespace.tmpl.partial @@ -0,0 +1,17 @@ +

{{>partials/title}}

+
{{{summary}}}
+
{{{conceptual}}}
+
{{{remarks}}}
+{{#children}} +

{{>partials/namespaceSubtitle}}

+ + {{#children}} + + + + {{/children}} +
+

+

{{{summary}}}

+
+{{/children}} diff --git a/eng/common/docgeneration/templates/matthews/styles/main.css b/eng/common/docgeneration/templates/matthews/styles/main.css new file mode 100644 index 00000000..944ef728 --- /dev/null +++ b/eng/common/docgeneration/templates/matthews/styles/main.css @@ -0,0 +1,311 @@ +@import url("https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.6.3/css/font-awesome.min.css"); + +/* Clickability fix for selector on sm devices */ +@media (min-width: 768px) and (max-width: 991px) { + article h1:first-of-type:before { + height: 0; + margin-top: 0; + } +} + +#search { + border: none; +} + +.fa-code { + font-size: 19px; +} + +.sidetoc, +body .toc, +.sidefilter, +.sidetoggle { + background-color: #f9fbe7; +} + +.sidenav, +.toc-toggle { + padding: 0; +} + +.sidetoggle { + padding-bottom: 15px; +} + +/* Remove center align from Navbar and Collapsible section */ +.collapse.in, +.collapsing { + text-align: unset; +} + +article h4 { + border-bottom: none; + line-height: normal; +} + +@media (min-width: 768px) { + .sidetoc, .sidefilter { + margin-left: -15px; + } +} + +@media (max-width: 767px) { + .navbar-collapse { + text-align: center !important; + } + + .navbar-collapse li .active { + border-radius: 20px; + } +} + +/* Collapsible Sections + ------------------------------------------------------- */ +.expander:after { + font-family: 'Glyphicons Halflings'; + content: "\e260"; + margin-left: 5px; + color: grey; + font-size: small; +} + +.expander.collapsed:after { + content: "\e259"; +} + +/* Floating buttons + ------------------------------------------------------- */ +.fab { + width: 40px; + height: 40px; + text-align: center; + padding: 11px 0 0 0; + border: none; + outline: none; + color: #FFF; + border-radius: 100%; + box-shadow: 0 3px 6px rgba(0,0,0,0.16), 0 3px 6px rgba(0,0,0,0.23); + transition:.3s; +} + +.fab:hover { + transform: scale(1.1); +} + +.fab + .fab { + margin-right: 15px; +} + +.contribution-panel { + z-index: 1000; + position: fixed; + right: 30px; + top: 70px; +} + +/* Bootstrap docs like sidebar + ------------------------------------------------------- */ +.affix h5 { + display: none; +} + +/* active & hover links */ +.affix ul > li > a:hover, +.affix ul > li.active > a, +.affix ul > li > a:focus { + color: #563d7c; + text-decoration: none; + background-color: transparent; + border-left-color: #563d7c; +} + +/* all active links */ +.affix ul > li.active > a, +.affix ul > li.active:hover > a, +.affix ul > li.active:focus >a { + font-weight: 700; +} + +/* nested active links */ +.affix ul ul > li.active > a, +.affix ul ul > li.active:hover > a, +.affix ul ul > li.active:focus > a { + font-weight: 500; +} + +/* all links */ +.affix ul > li > a { + color: #999; + border-left: 2px solid transparent; + padding: 4px 20px; + font-size: 13px; + font-weight: 400; +} + +/* nested links */ +.affix ul ul > li > a { + padding-top: 1px; + padding-bottom: 1px; + padding-left: 30px; + font-size: 12px; +} + +/* hide inactive nested list */ +.affix ul ul { + display: none; +} + +/* show active nested list */ +.affix ul > li.active > ul { + display: block; +} + +.affix > ul > li > a:before { + content: ''; +} + +.affix ul ul > li > a:before { + content: ''; +} + +/* Style Buttons + ------------------------------------------------------- */ +.btn-warning { + background-color: #0071c5; +} + +.btn-info { + background-color: #0071c5; +} + +/* Navbar Hamburger + ------------------------------------------------------- */ +.icon-bar { + transition: 0.4s; +} + +/* Rotate first bar */ +.change .icon-bar:nth-of-type(2) { + transform: rotate(-45deg) translate(-4px, 5px) ; +} + +/* Fade out the second bar */ +.change .icon-bar:nth-of-type(3) { + opacity: 0; +} + +/* Rotate last bar */ +.change .icon-bar:nth-of-type(4) { + transform: rotate(45deg) translate(-4px, -5px) ; +} + +/* Custom Navbar + ------------------------------------------------------- */ +.navbar-inverse { + background-color: #0071c5; + opacity: 0.95; + border-color: #0071c5; +} +.navbar-inverse .navbar-brand { + color: #ffffff; +} +.navbar-inverse .navbar-brand:hover, +.navbar-inverse .navbar-brand:focus { + color: #ecdbff; +} +.navbar-inverse .navbar-text { + color: #ffffff; +} +.navbar-inverse .navbar-nav > li > a { + color: #ffffff; +} +.navbar-inverse .navbar-nav > li > a:hover, +.navbar-inverse .navbar-nav > li > a:focus { + color: #ecdbff; +} +.navbar-inverse .navbar-nav > .active > a, +.navbar-inverse .navbar-nav > .active > a:hover, +.navbar-inverse .navbar-nav > .active > a:focus { + color: #ecdbff; + background-color: #0071c5; +} +.navbar-inverse .navbar-nav > .open > a, +.navbar-inverse .navbar-nav > .open > a:hover, +.navbar-inverse .navbar-nav > .open > a:focus { + color: #ecdbff; + background-color: #0071c5; +} +.navbar-inverse .navbar-toggle { + border-color: #0071c5; +} +.navbar-inverse .navbar-toggle:hover, +.navbar-inverse .navbar-toggle:focus { + background-color: #0071c5; +} +.navbar-inverse .navbar-toggle .icon-bar { + background-color: #ffffff; +} +.navbar-inverse .navbar-collapse, +.navbar-inverse .navbar-form { + border: none; +} +.navbar-inverse .navbar-link { + color: #ffffff; +} +.navbar-inverse .navbar-link:hover { + color: #ecdbff; +} +.versionarrow { + margin-left: 0.8em; + margin-top: -1.5em; + margin-bottom: -1em; + padding: 1em; +} + +.versionarrow::before { + position: absolute; + content: ''; + width: 0; + height: 0; + border: .5em solid transparent; + border-left-color: gray; + transform-origin: 0 50%; + transition: transform .1s; + margin-top: 0.2em; +} + + +.versionarrow.disable { + text-decoration: line-through; +} + +.versionarrow.down::before { + transform: rotate(90deg); + margin-top: 0em; + transition: transform .1s; +} + +@media (max-width: 767px) { + .navbar-inverse .navbar-nav .open .dropdown-menu > li > a { + color: #ffffff; + } + .navbar-inverse .navbar-nav .open .dropdown-menu > li > a:hover, + .navbar-inverse .navbar-nav .open .dropdown-menu > li > a:focus { + color: #ecdbff; + } + .navbar-inverse .navbar-nav .open .dropdown-menu > .active > a, + .navbar-inverse .navbar-nav .open .dropdown-menu > .active > a:hover, + .navbar-inverse .navbar-nav .open .dropdown-menu > .active > a:focus { + color: #ecdbff; + background-color: #0071c5; + } +} + +.navbar-version-select { + padding: 2px; + border: none; + border-radius: 2px; + box-shadow: none; + -webkit-appearance: media-time-remaining-display; + margin-top: 14px; +} \ No newline at end of file diff --git a/eng/common/docgeneration/templates/matthews/styles/main.js b/eng/common/docgeneration/templates/matthews/styles/main.js new file mode 100644 index 00000000..7ab1dd9e --- /dev/null +++ b/eng/common/docgeneration/templates/matthews/styles/main.js @@ -0,0 +1,238 @@ +// Use container fluid +var containers = $(".container"); +containers.removeClass("container"); +containers.addClass("container-fluid"); + +WINDOW_CONTENTS = window.location.href.split('/') +var SELECTED_LANGUAGE = '' +var PACKAGE_REGEX = '' +var PACKAGE_REPLACEMENT = '' + +ATTR1 = '[System.ComponentModel.EditorBrowsable]\n<' + +// Navbar Hamburger +$(function () { + $(".navbar-toggle").click(function () { + $(this).toggleClass("change"); + }) +}) + +// Select list to replace affix on small screens +$(function () { + var navItems = $(".sideaffix .level1 > li"); + + if (navItems.length == 0) { + return; + } + + var selector = $("