From 977049cf32be72fdb0f84c4450ad73ef87a9b5dd Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Tue, 26 Mar 2024 11:46:55 -0700 Subject: [PATCH] Use 1es pipeline templates (#4920) Co-authored-by: Scott Beddall (from Dev Box) Co-authored-by: Timothee Guerin --- eng/pipelines/ci.yaml | 144 ++++++++++-------- eng/pipelines/publish-dev.yaml | 42 ++--- eng/pipelines/publish.yaml | 124 ++++++++------- eng/pipelines/templates/1es-redirect.yml | 55 +++++++ .../templates/build-and-push-docker.yaml | 8 +- eng/pipelines/templates/variables/image.yml | 24 +++ 6 files changed, 251 insertions(+), 146 deletions(-) create mode 100644 eng/pipelines/templates/1es-redirect.yml create mode 100644 eng/pipelines/templates/variables/image.yml diff --git a/eng/pipelines/ci.yaml b/eng/pipelines/ci.yaml index fa8ddd39fb..1e9c7e0028 100644 --- a/eng/pipelines/ci.yaml +++ b/eng/pipelines/ci.yaml @@ -6,80 +6,94 @@ trigger: - main -pool: - vmImage: "ubuntu-latest" +extends: + template: /eng/pipelines/templates/1es-redirect.yml + parameters: + stages: + - stage: main + displayName: Main + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + jobs: + - job: main + displayName: "Build and Unit test" + steps: + - template: /eng/pipelines/templates/build.yaml -stages: - - stage: main - displayName: Main - jobs: - - job: main - displayName: "Build and Unit test" - steps: - - template: ./templates/build.yaml + - script: npx @microsoft/rush test:ci -v + displayName: Test - - script: npx @microsoft/rush test:ci -v - displayName: Test + - task: PublishCodeCoverageResults@1 + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: $(Build.SourcesDirectory)/packages/**/coverage/cobertura-coverage.xml + reportDirectory: $(Build.SourcesDirectory)/packages/**/coverage - - task: PublishCodeCoverageResults@1 - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: $(Build.SourcesDirectory)/packages/**/coverage/cobertura-coverage.xml - reportDirectory: $(Build.SourcesDirectory)/packages/**/coverage + - script: npx @microsoft/rush check-format + displayName: Check formatting - - script: npx @microsoft/rush check-format - displayName: Check formatting + - script: npx @microsoft/rush lint -v + displayName: Lint - - script: npx @microsoft/rush lint -v - displayName: Lint + - script: npx @microsoft/rush regen-codemodel + displayName: Regen codemodel schema - - script: npx @microsoft/rush regen-codemodel - displayName: Regen codemodel schema + - script: node ./eng/scripts/check-for-changed-files.js + displayName: Check Git Status For Changed Files - - script: node ./eng/scripts/check-for-changed-files.js - displayName: Check Git Status For Changed Files + - script: | + commitId=$(git rev-parse --short HEAD) + npx @microsoft/rush publish --apply --prerelease-name="ci.$commitId" + displayName: Bump versions - - script: | - commitId=$(git rev-parse --short HEAD) - npx @microsoft/rush publish --apply --prerelease-name="ci.$commitId" - displayName: Bump versions + - script: npx @microsoft/rush publish --publish --pack --include-all + displayName: Pack packages - - script: npx @microsoft/rush publish --publish --pack --include-all - displayName: Pack packages + - task: 1ES.PublishPipelineArtifact@1 + inputs: + targetPath: $(Build.SourcesDirectory)/common/temp/artifacts/packages + artifact: packages + displayName: Publish packages(.tgz) as pipeline artifacts - - publish: $(Build.SourcesDirectory)/common/temp/artifacts/packages - artifact: packages - displayName: Publish packages(.tgz) as pipeline artifacts + - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - template: /eng/pipelines/templates/create-tryit-comment.yaml + # - job: lintdocs + # displayName: Lint docs + # continueOnError: true + # steps: + # - script: | + # sudo npm install -g markdown-link-check + # find docs -name "*.md" -print0 | xargs -0 -n1 markdown-link-check -c ./markdown-link-check-config.json + # displayName: Find - - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - - template: ./templates/create-tryit-comment.yaml - # - job: lintdocs - # displayName: Lint docs - # continueOnError: true - # steps: - # - script: | - # sudo npm install -g markdown-link-check - # find docs -name "*.md" -print0 | xargs -0 -n1 markdown-link-check -c ./markdown-link-check-config.json - # displayName: Find + - stage: regressiontests + displayName: Regression Tests + dependsOn: main + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + jobs: + - template: /eng/pipelines/templates/regression-tests.yaml + parameters: + name: node18 + nodeVersion: 18.x + - template: /eng/pipelines/templates/regression-tests.yaml + parameters: + name: node20 + nodeVersion: 20.x - - stage: regressiontests - displayName: Regression Tests - dependsOn: main - jobs: - - template: ./templates/regression-tests.yaml - parameters: - name: node18 - nodeVersion: 18.x - - template: ./templates/regression-tests.yaml - parameters: - name: node20 - nodeVersion: 20.x - - - stage: smoketests - displayName: Smoke Tests - dependsOn: main - jobs: - - template: ./templates/smoke-tests.yaml - parameters: - name: node20 - nodeVersion: 20.x + - stage: smoketests + displayName: Smoke Tests + dependsOn: main + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + jobs: + - template: /eng/pipelines/templates/smoke-tests.yaml + parameters: + name: node20 + nodeVersion: 20.x diff --git a/eng/pipelines/publish-dev.yaml b/eng/pipelines/publish-dev.yaml index 3e0a8dec54..fdf709337f 100644 --- a/eng/pipelines/publish-dev.yaml +++ b/eng/pipelines/publish-dev.yaml @@ -5,25 +5,29 @@ trigger: none -pool: - vmImage: "ubuntu-latest" +extends: + template: /eng/pipelines/templates/1es-redirect.yml + parameters: + stages: + - stage: npm + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + jobs: + - job: dev + displayName: Publish dev + steps: + - template: /eng/pipelines/templates/build.yaml -stages: - - stage: npm - jobs: - - job: dev - displayName: Publish dev - steps: - - template: ./templates/build.yaml + - script: | + commitId=$(git rev-parse --short HEAD) + npx @microsoft/rush publish --apply --prerelease-name="ci.$commitId" + displayName: Bump versions - - script: | - commitId=$(git rev-parse --short HEAD) - npx @microsoft/rush publish --apply --prerelease-name="ci.$commitId" - displayName: Bump versions + - script: npx @microsoft/rush publish --publish --pack --include-all + displayName: Pack packages - - script: npx @microsoft/rush publish --publish --pack --include-all - displayName: Pack packages - - - script: | - npx @microsoft/rush publish --publish --registry https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-tools-feeds/npm/registry --npm-auth-token "$(azure-sdk-devops-npm-token)" --include-all --set-access-level public - displayName: Publish packages + - script: | + npx @microsoft/rush publish --publish --registry https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-tools-feeds/npm/registry --npm-auth-token "$(azure-sdk-devops-npm-token)" --include-all --set-access-level public + displayName: Publish packages diff --git a/eng/pipelines/publish.yaml b/eng/pipelines/publish.yaml index bb8418cd10..fd8c4be832 100644 --- a/eng/pipelines/publish.yaml +++ b/eng/pipelines/publish.yaml @@ -7,66 +7,74 @@ trigger: - main - release/* -pool: - vmImage: "ubuntu-latest" +extends: + template: /eng/pipelines/templates/1es-redirect.yml + parameters: + stages: + - stage: npm + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + jobs: + - job: stable + displayName: Publish stable + steps: + - template: /eng/pipelines/templates/build.yaml -stages: - - stage: npm - jobs: - - job: stable - displayName: Publish stable - steps: - - template: ./templates/build.yaml + - script: npx @microsoft/rush publish --publish --pack --include-all + displayName: Pack packages - - script: npx @microsoft/rush publish --publish --pack --include-all - displayName: Pack packages + - script: | + NPM_AUTH_TOKEN="$(azure-sdk-npm-token)" npx @microsoft/rush publish --publish --include-all --set-access-level public + displayName: Publish packages - - script: | - NPM_AUTH_TOKEN="$(azure-sdk-npm-token)" npx @microsoft/rush publish --publish --include-all --set-access-level public - displayName: Publish packages + - job: prerelease + displayName: Publish prerelease + dependsOn: stable # Publish prerelease after to prevent race condition. + steps: + - template: /eng/pipelines/templates/build.yaml - - job: prerelease - displayName: Publish prerelease - dependsOn: stable # Publish prerelease after to prevent race condition. - steps: - - template: ./templates/build.yaml + - script: node ./eng/bump-for-prerelease.js + displayName: Bump version to prerelease targets - - script: node ./eng/bump-for-prerelease.js - displayName: Bump version to prerelease targets - - - script: | - NPM_AUTH_TOKEN="$(azure-sdk-npm-token)" npx @microsoft/rush publish --publish --include-all --set-access-level public --tag next - displayName: Publish packages - - stage: Docker - dependsOn: npm - jobs: - - template: ./templates/build-and-push-docker.yaml - parameters: - name: base - image: azsdkengsys.azurecr.io/azuresdk/autorest - dockerFolder: ./docker/base/ubuntu - readmePath: docker/base/readme.md - - template: ./templates/build-and-push-docker.yaml - parameters: - name: dotnet - image: azsdkengsys.azurecr.io/azuresdk/autorest-dotnet - dockerFolder: ./docker/dotnet/ubuntu - readmePath: docker/dotnet/readme.md - - template: ./templates/build-and-push-docker.yaml - parameters: - name: python - image: azsdkengsys.azurecr.io/azuresdk/autorest-python - dockerFolder: ./docker/python/ubuntu - readmePath: docker/python/readme.md - - template: ./templates/build-and-push-docker.yaml - parameters: - name: java - image: azsdkengsys.azurecr.io/azuresdk/autorest-java - dockerFolder: ./docker/java/ubuntu - readmePath: docker/java/readme.md - - template: ./templates/build-and-push-docker.yaml - parameters: - name: all - image: azsdkengsys.azurecr.io/azuresdk/autorest-all - dockerFolder: ./docker/all/ubuntu - readmePath: docker/all/readme.md + - script: | + NPM_AUTH_TOKEN="$(azure-sdk-npm-token)" npx @microsoft/rush publish --publish --include-all --set-access-level public --tag next + displayName: Publish packages + - stage: Docker + dependsOn: npm + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + jobs: + - template: /eng/pipelines/templates/build-and-push-docker.yaml + parameters: + name: base + image: azsdkengsys.azurecr.io/azuresdk/autorest + dockerFolder: ./docker/base/ubuntu + readmePath: docker/base/readme.md + - template: /eng/pipelines/templates/build-and-push-docker.yaml + parameters: + name: dotnet + image: azsdkengsys.azurecr.io/azuresdk/autorest-dotnet + dockerFolder: ./docker/dotnet/ubuntu + readmePath: docker/dotnet/readme.md + - template: /eng/pipelines/templates/build-and-push-docker.yaml + parameters: + name: python + image: azsdkengsys.azurecr.io/azuresdk/autorest-python + dockerFolder: ./docker/python/ubuntu + readmePath: docker/python/readme.md + - template: /eng/pipelines/templates/build-and-push-docker.yaml + parameters: + name: java + image: azsdkengsys.azurecr.io/azuresdk/autorest-java + dockerFolder: ./docker/java/ubuntu + readmePath: docker/java/readme.md + - template: /eng/pipelines/templates/build-and-push-docker.yaml + parameters: + name: all + image: azsdkengsys.azurecr.io/azuresdk/autorest-all + dockerFolder: ./docker/all/ubuntu + readmePath: docker/all/readme.md diff --git a/eng/pipelines/templates/1es-redirect.yml b/eng/pipelines/templates/1es-redirect.yml new file mode 100644 index 0000000000..beb2f1c7c6 --- /dev/null +++ b/eng/pipelines/templates/1es-redirect.yml @@ -0,0 +1,55 @@ +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +parameters: + - name: stages + type: stageList + default: [] + - name: variables + type: object + default: {} + - name: Use1ESOfficial + type: boolean + default: true + +extends: + ${{ if and(parameters.Use1ESOfficial, eq(variables['System.TeamProject'], 'internal')) }}: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + ${{ else }}: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + settings: + skipBuildTagsForGitHubPullRequests: true + sdl: + git: + longpaths: true + sourceAnalysisPool: + name: azsdk-pool-mms-win-2022-general + image: azsdk-pool-mms-win-2022-1espt + os: windows + psscriptanalyzer: + compiled: true + break: true + stages: + - ${{ each stage in parameters.stages }}: + - ${{ each pair in stage }}: + ${{ if ne(pair.key, 'variables') }}: + ${{ pair.key }}: ${{ pair.value }} + variables: + - template: /eng/pipelines/templates/variables/image.yml@self + - ${{ each variable in parameters.variables }}: + - ${{ if eq(variable.key, 'template') }}: + - template: ${{ variable.value }} + - ${{ else }}: + - name: ${{ coalesce(variable.key, variable.name) }} + value: ${{ variable.value }} + - ${{ each variable in stage.variables }}: + - ${{ if in(variable.key, 'template', 'group') }}: + - ${{ variable.key }}: ${{ variable.value }} + - ${{ else }}: + - name: ${{ coalesce(variable.key, variable.name) }} + value: ${{ variable.value }} diff --git a/eng/pipelines/templates/build-and-push-docker.yaml b/eng/pipelines/templates/build-and-push-docker.yaml index 1dff0fab3e..3e7934b9ff 100644 --- a/eng/pipelines/templates/build-and-push-docker.yaml +++ b/eng/pipelines/templates/build-and-push-docker.yaml @@ -27,7 +27,7 @@ jobs: docker build -t ${{ parameters.image }} . displayName: Build image workingDirectory: ${{ parameters.dockerFolder }} - - - script: | - docker push ${{ parameters.image }} - displayName: Publish image + templateContext: + outputs: + - output: containerImage + image: ${{ parameters.image }} diff --git a/eng/pipelines/templates/variables/image.yml b/eng/pipelines/templates/variables/image.yml new file mode 100644 index 0000000000..f24cec2a17 --- /dev/null +++ b/eng/pipelines/templates/variables/image.yml @@ -0,0 +1,24 @@ +variables: + - name: LINUXPOOL + value: azsdk-pool-mms-ubuntu-2004-general + - name: WINDOWSPOOL + value: azsdk-pool-mms-win-2022-general + - name: MACPOOL + value: Azure Pipelines + + - name: LINUXVMIMAGE + value: azsdk-pool-mms-ubuntu-2004-1espt + - name: LINUXNEXTVMIMAGE + value: azsdk-pool-mms-ubuntu-2204-1espt + - name: WINDOWSVMIMAGE + value: azsdk-pool-mms-win-2022-1espt + - name: MACVMIMAGE + value: macos-11 + + # Values required for pool.os field in 1es pipeline templates + - name: LINUXOS + value: linux + - name: WINDOWSOS + value: windows + - name: MACOS + value: macOS