Merge branch 'Azure:main' into main

.github/CODEOWNERS

@@ -964,6 +964,12 @@
 # ServiceLabel: %Device Registry %Mgmt
 # ServiceOwners:                                                   @davidemontanari @atastrophic @marcodalessandro @rohankhandelwal @riteshrao
 
+# PRLabel: %DevOps - Infrastructure
+/sdk/devopsinfrastructure/Azure.ResourceManager.*/                 @oolaoluwa
+
+# ServiceLabel: %DevOps - Infrastructure
+# ServiceOwners:                                                   @oolaoluwa
+
 # PRLabel: %Informatica Data Management
 /sdk/informaticadatamanagement/Azure.ResourceManager.*/            @almasak
 

.github/workflows/event.yml

@@ -18,7 +18,7 @@ jobs:
       pull-requests: write # to read pull requests and write comments (azure/azure-sdk-actions)
       checks: read # to read check status (azure/azure-sdk-actions)
     name: Handle ${{ github.event_name }} ${{ github.event.action }} event
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest # This image is intentionally set to "latest", and not to a specific version
     steps:
       - uses: azure/azure-sdk-actions@main
         with:

.vscode/cspell.json

@@ -410,6 +410,13 @@
         "liftr"
       ]
     },
+    {
+      "filename": "**/sdk/devopsinfrastructure/**/*.cs",
+      "words": [
+        "Reimage",
+        "Reimaging"
+      ]
+    },
     {
       "filename": "**/sdk/newrelic/**/*.cs",
       "words": [

doc/DataPlaneCodeGeneration/Autorest_DataPlane_Quickstart.md

@@ -34,17 +34,17 @@ This tutorial has following sections:
 - Install Visual Studio 2022 (Community or higher) and make sure you have the [latest updates](https://www.visualstudio.com/).
   - Need at least .NET Framework 4.6.1 and 4.7 development tools
   - Install the **.NET Core cross-platform development** workloads in Visual Studio
-- Install **.NET 6.0 SDK** for your specific platform. (or a higher version)
+- Install **.NET 8.0 SDK** for your specific platform. (or a higher version)
 - Install the latest version of [git](https://git-scm.com/downloads)
 - Install [PowerShell](https://docs.microsoft.com/powershell/scripting/install/installing-powershell), version 7 or higher.
-- Install [NodeJS](https://nodejs.org/) (14.x.x).
+- Install [NodeJS](https://nodejs.org/) (18.x.x).
 
 ## Setup your repository
 
 - Fork and clone an [azure-sdk-for-net](https://github.com/Azure/azure-sdk-for-net) repo. Follow the instructions in the [.NET CONTRIBUTING.md](https://github.com/Azure/azure-sdk-for-net/issues/12903) to fork and clone the `azure-sdk-for-net` repo.
 - Create a branch to work in.
 
-## Create starter package  
+## Create starter package
 
 For this guide, we'll create a getting started project in a branch of your fork of `azure-sdk-for-net` repo. The started project will be under `sdk\<servie name>\<package name>` directory of `azure-sdk-for-net` repo. The package will contain several folders and files (see following). Please refer to [sdk-directory-layout](https://github.com/Azure/azure-sdk/blob/main/docs/policies/repostructure.md#sdk-directory-layout) for detail information.
 
@@ -59,7 +59,7 @@ sdk\<service name>\<package name>\CHANGELOG.md
 
 - `<service name>` - Should be the short name for the azure service. e.g. deviceupdate
 - `<package name>` -  Should be the name of the shipping package, or an abbreviation that distinguishes the given shipping artifact for the given service. It will be `Azure.<group>.<service>`, e.g. Azure.IoT.DeviceUpdate
-  
+
 We will use dotnet project template [Azure.Template](https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/template/Azure.Template) to automatically create the project.
 
 You can run `eng\scripts\automation\Invoke-DataPlaneGenerateSDKPackage.ps1` to generate the starting SDK client library package directly as following:

doc/DataPlaneCodeGeneration/AzureSDKGeneration_Prerequistites.md

@@ -5,10 +5,10 @@
 - Install Visual Studio 2022 (Community or higher) and make sure you have the [latest updates](https://www.visualstudio.com/).
   - Need at least .NET Framework 4.6.1 and 4.7 development tools
   - Install the **.NET Core cross-platform development** workloads in Visual Studio
-- Install **.NET 6.0 SDK** for your specific platform. (or a higher version)
+- Install **.NET 8.0 SDK** for your specific platform. (or a higher version)
 - Install the latest version of [git](https://git-scm.com/downloads)
 - Install [PowerShell](https://docs.microsoft.com/powershell/scripting/install/installing-powershell), version 7 or higher.
-- Install [NodeJS](https://nodejs.org/) (16.x.x or above).
+- Install [NodeJS](https://nodejs.org/) (18.x.x or above).
 
 ## Set up TypeSpec project
 

eng/Packages.Data.props

@@ -116,14 +116,14 @@
     <PackageReference Update="Azure.Messaging.EventHubs" Version="5.11.5" />
     <PackageReference Update="Azure.Messaging.EventGrid" Version="4.21.0" />
     <PackageReference Update="Azure.Messaging.ServiceBus" Version="7.18.1" />
-    <PackageReference Update="Azure.Messaging.WebPubSub" Version="1.3.0" />
+    <PackageReference Update="Azure.Messaging.WebPubSub" Version="1.4.0" />
     <PackageReference Update="Azure.MixedReality.Authentication" version= "1.2.0" />
     <PackageReference Update="Azure.Monitor.OpenTelemetry.Exporter" Version="1.4.0-beta.1" />
     <PackageReference Update="Azure.Monitor.Query" Version="1.1.0" />
     <PackageReference Update="Azure.Identity" Version="1.12.0" />
-    <PackageReference Update="Azure.Security.KeyVault.Secrets" Version="4.2.0" />
-    <PackageReference Update="Azure.Security.KeyVault.Keys" Version="4.2.0" />
-    <PackageReference Update="Azure.Security.KeyVault.Certificates" Version="4.2.0" />
+    <PackageReference Update="Azure.Security.KeyVault.Secrets" Version="4.6.0" />
+    <PackageReference Update="Azure.Security.KeyVault.Keys" Version="4.6.0" />
+    <PackageReference Update="Azure.Security.KeyVault.Certificates" Version="4.6.0" />
     <PackageReference Update="Azure.Storage.Common" Version="12.20.1" />
     <PackageReference Update="Azure.Storage.Blobs" Version="12.21.1" />
     <PackageReference Update="Azure.Storage.Queues" Version="12.19.1" />
@@ -187,7 +187,7 @@
     These are dependencies for Track 2 integration packages
   -->
   <ItemGroup Condition="'$(IsClientLibrary)' == 'true' and $(MSBuildProjectName.StartsWith('Microsoft.'))">
-    <PackageReference Update="Apache.Avro" Version="1.11.0" />
+    <PackageReference Update="Apache.Avro" Version="1.12.0" />
     <PackageReference Update="CloudNative.CloudEvents" Version="2.0.0" />
     <PackageReference Update="CloudNative.CloudEvents.SystemTextJson" Version="2.0.0" />
     <PackageReference Update="Google.Protobuf" Version="3.24.3" />
@@ -227,8 +227,8 @@
     All should have PrivateAssets="All" set so they don't become package dependencies
   -->
   <ItemGroup>
-    <PackageReference Update="Microsoft.Azure.AutoRest.CSharp" Version="3.0.0-beta.20240808.4" PrivateAssets="All" />
-    <PackageReference Update="Azure.ClientSdk.Analyzers" Version="0.1.1-dev.20240429.1" PrivateAssets="All" />
+    <PackageReference Update="Microsoft.Azure.AutoRest.CSharp" Version="3.0.0-beta.20240818.1" PrivateAssets="All" />
+    <PackageReference Update="Azure.ClientSdk.Analyzers" Version="0.1.1-dev.20240813.2" PrivateAssets="All" />
     <PackageReference Update="coverlet.collector" Version="3.2.0" PrivateAssets="All" />
     <PackageReference Update="Microsoft.CodeAnalysis.NetAnalyzers" Version="8.0.0" PrivateAssets="All" />
     <PackageReference Update="Microsoft.CodeAnalysis.BannedApiAnalyzers" Version="3.3.2" PrivateAssets="All" />
@@ -267,8 +267,8 @@
     <PackageReference Update="Azure.ResourceManager.EventHubs" Version="1.0.0" />
     <PackageReference Update="Azure.ResourceManager.ContainerRegistry" Version="1.1.0" />
     <PackageReference Update="Azure.Search.Documents" Version="11.2.0" />
-    <PackageReference Update="Azure.Security.KeyVault.Keys" Version="4.4.0" />
-    <PackageReference Update="Azure.Security.KeyVault.Secrets" Version="4.2.0-beta.4" />
+    <PackageReference Update="Azure.Security.KeyVault.Keys" Version="4.6.0" />
+    <PackageReference Update="Azure.Security.KeyVault.Secrets" Version="4.6.0" />
     <PackageReference Update="Azure.Storage.Blobs" Version="12.20.0" />
     <PackageReference Update="Azure.Storage.Files.DataLake" Version="12.8.0" />
     <PackageReference Update="BenchmarkDotNet" Version="0.13.4" />

eng/common/pipelines/templates/steps/eng-common-workflow-enforcer.yml

@@ -21,7 +21,7 @@ steps:
         }
         if ((!"$(System.PullRequest.SourceBranch)".StartsWith("sync-.github/workflows")) -and "$(System.PullRequest.TargetBranch)" -match "^(refs/heads/)?$(DefaultBranch)$")
         {
-          $filesInCommonDir  = & "eng/common/scripts/get-changedfiles.ps1" -DiffPath '.github/workflows/*' -DiffFilterType ""
+          $filesInCommonDir  = & "eng/common/scripts/get-changedfiles.ps1" -DiffPath '.github/workflows/*event*' -DiffFilterType ""
           if (($LASTEXITCODE -eq 0) -and ($filesInCommonDir.Count -gt 0))
           {
             Write-Host "##vso[task.LogIssue type=error;]Changes to files under '.github/workflows' directory should not be made in this Repo`n${filesInCommonDir}"

eng/common/pipelines/templates/steps/publish-blobs.yml

@@ -1,20 +1,14 @@
 parameters:
   FolderForUpload: ''
   TargetLanguage: ''
-  BlobName: ''
-  ScriptPath: ''
+  BlobName: '' # To be removed once all usages are removed in the languages
+  BlobAccountName: 'https://azuresdkdocs.blob.core.windows.net'
+  ScriptPath: 'eng/common/scripts/copy-docs-to-blobstorage.ps1'
   ArtifactLocation: ''
   RepoId: $(Build.Repository.Name)
 
 steps:
 - template: /eng/common/pipelines/templates/steps/set-default-branch.yml
-- pwsh: |
-    if (!(Test-Path '$(Build.BinariesDirectory)/azcopy/azcopy_windows_amd64_*/azcopy.exe')) {
-      Invoke-WebRequest -MaximumRetryCount 10 -Uri "https://aka.ms/downloadazcopy-v10-windows" -OutFile "azcopy.zip" | Wait-Process;
-      Expand-Archive -Path "azcopy.zip" -DestinationPath "$(Build.BinariesDirectory)/azcopy/" -Force
-    }
-  workingDirectory: $(Build.BinariesDirectory)
-  displayName: Download and Extract azcopy Zip
 
 - task: AzurePowerShell@5
   displayName: 'Copy Docs to Blob Storage'
@@ -24,9 +18,8 @@ steps:
     ScriptType: 'FilePath'
     ScriptPath: ${{ parameters.ScriptPath }}
     ScriptArguments: >
-      -AzCopy $(Resolve-Path "$(Build.BinariesDirectory)/azcopy/azcopy_windows_amd64_*/azcopy.exe")[0]
       -DocLocation "${{ parameters.FolderForUpload }}"
-      -BlobName "${{ parameters.BlobName }}"
+      -BlobAccountName "${{ parameters.BlobAccountName }}"
       -PublicArtifactLocation "${{ parameters.ArtifactLocation }}"
       -RepoReplaceRegex "(https://github.com/${{ parameters.RepoId }}/(?:blob|tree)/)$(DefaultBranch)"
     azurePowerShellVersion: latestVersion

eng/common/scripts/Delete-RemoteBranches.ps1

@@ -15,7 +15,6 @@ param(
   $AuthToken
 )
 Set-StrictMode -version 3
-
 . (Join-Path $PSScriptRoot common.ps1)
 
 function Get-AllBranchesAndPullRequestInfo($owner, $repo) {
@@ -66,67 +65,124 @@ if ($AuthToken) {
 }
 
 $owner, $repo = $RepoId -split "/"
-$branches = Get-AllBranchesAndPullRequestInfo $owner $repo
 
-foreach ($branch in $branches)
-{
-  $branchName = $branch.Name
-  if ($branchName -notmatch $BranchRegex) {
-    continue
+# These will always be output at the end of the script. Their only purpose is for information gathering
+# Total number returned from query
+$totalBranchesFromQuery = 0
+# reasons why a branch was skipped
+$skippedBranchNotMatchRegex = 0
+$skippedForCommitDate = 0
+$skippedForOpenPRs = 0
+$skippedForPRNotInBranch = 0
+$skippedForPRNotInRepo = 0
+# gh call counters
+$ghPRViewCalls = 0
+$ghBranchDeleteCalls = 0
+
+try {
+  # Output the core rate limit at the start of processing. There's no real need
+  # to output this at the end because the GH call counts are being output
+  $coreRateLimit = Get-RateLimit core
+  Write-RateLimit $coreRateLimit
+  # Output the GraphQL rate limit before and after the call
+  $graphqlRateLimit = Get-RateLimit graphql
+  Write-RateLimit $graphqlRateLimit "Before GraphQL Call"
+  $branches = Get-AllBranchesAndPullRequestInfo $owner $repo
+  $graphqlRateLimit = Get-RateLimit graphql
+  Write-RateLimit $graphqlRateLimit "After GraphQL Call"
+
+  if ($branches) {
+    $totalBranchesFromQuery = $branches.Count
   }
-  $openPullRequests = @($branch.pullRequests | Where-Object { !$_.Closed })
 
-  # If we have a central PR that created this branch still open don't delete the branch
-  if ($CentralRepoId)
+  foreach ($branch in $branches)
   {
-    $pullRequestNumber = $matches["PrNumber"]
-    # If central PR number is not found, then skip
-    if (!$pullRequestNumber) {
-      LogError "No PR number found in the branch name. Please check the branch name '$branchName'. Skipping..."
+    $branchName = $branch.Name
+    if ($branchName -notmatch $BranchRegex) {
+      $skippedBranchNotMatchRegex++
       continue
     }
+    $openPullRequests = @($branch.pullRequests | Where-Object { !$_.Closed })
 
-    $centralPR = gh pr view --json 'url,closed' --repo $CentralRepoId $pullRequestNumber | ConvertFrom-Json
-    if ($LASTEXITCODE) {
-      LogError "PR '$pullRequestNumber' not found in repo '$CentralRepoId'. Skipping..."
-      continue;
-    } else {
-      LogDebug "Found central PR $($centralPR.url) and Closed=$($centralPR.closed)"
-      if (!$centralPR.Closed) {
-        # Skipping if there is an open central PR open for the branch.
-        LogDebug "Central PR is still open so skipping the deletion of branch '$branchName'. Skipping..."
-        continue;
+    # If we have a central PR that created this branch still open don't delete the branch
+    if ($CentralRepoId)
+    {
+      $pullRequestNumber = $matches["PrNumber"]
+      # If central PR number is not found, then skip
+      if (!$pullRequestNumber) {
+        LogError "No PR number found in the branch name. Please check the branch name '$branchName'. Skipping..."
+        $skippedForPRNotInBranch++
+        continue
+      }
+
+      $ghPRViewCalls++
+      $centralPR = gh pr view --json 'url,closed' --repo $CentralRepoId $pullRequestNumber | ConvertFrom-Json
+      if ($LASTEXITCODE) {
+        LogError "PR '$pullRequestNumber' not found in repo '$CentralRepoId'. Skipping..."
+        $skippedForPRNotInRepo++
+        continue
+      } else {
+        LogDebug "Found central PR $($centralPR.url) and Closed=$($centralPR.closed)"
+        if (!$centralPR.Closed) {
+          $skippedForOpenPRs++
+          # Skipping if there is an open central PR open for the branch.
+          LogDebug "Central PR is still open so skipping the deletion of branch '$branchName'. Skipping..."
+          continue
+        }
       }
     }
-  }
-  else {
-    # Not CentralRepoId - not associated with a central repo PR
-    if ($openPullRequests.Count -gt 0 -and !$DeleteBranchesEvenIfThereIsOpenPR) {
-      LogDebug "Found open PRs associate with branch '$branchName'. Skipping..."
-      continue
+    else {
+      # Not CentralRepoId - not associated with a central repo PR
+      if ($openPullRequests.Count -gt 0 -and !$DeleteBranchesEvenIfThereIsOpenPR) {
+        $skippedForOpenPRs++
+        LogDebug "Found open PRs associate with branch '$branchName'. Skipping..."
+        continue
+      }
     }
-  }
 
-  # If there is date filter, then check if branch last commit is older than the date.
-  if ($LastCommitOlderThan)
-  {
-    $commitDate = $branch.committedDate
-    if ($commitDate -gt $LastCommitOlderThan) {
-      LogDebug "The branch $branch last commit date '$commitDate' is newer than the date '$LastCommitOlderThan'. Skipping..."
-      continue
+    # If there is date filter, then check if branch last commit is older than the date.
+    if ($LastCommitOlderThan)
+    {
+      $commitDate = $branch.committedDate
+      if ($commitDate -gt $LastCommitOlderThan) {
+        $skippedForCommitDate++
+        LogDebug "The branch $branch last commit date '$commitDate' is newer than the date '$LastCommitOlderThan'. Skipping..."
+        continue
+      }
     }
-  }
 
-  foreach ($openPullRequest in $openPullRequests) {
-    Write-Host "Note: Open pull Request '$($openPullRequest.url)' will be closed after branch deletion, given the central PR is closed."
-  }
+    foreach ($openPullRequest in $openPullRequests) {
+      LogDebug "Note: Open pull Request '$($openPullRequest.url)' will be closed after branch deletion, given the central PR is closed."
+    }
 
-  $commitUrl = $branch.commitUrl
-  if ($PSCmdlet.ShouldProcess("'$branchName' in '$RepoId'", "Deleting branch on cleanup script")) {
-    gh api "repos/${RepoId}/git/refs/heads/${branchName}" -X DELETE
-    if ($LASTEXITCODE) {
-      LogError "Deletion of branch '$branchName` failed"
+    $commitUrl = $branch.commitUrl
+    if ($PSCmdlet.ShouldProcess("'$branchName' in '$RepoId'", "Deleting branch on cleanup script")) {
+      $ghBranchDeleteCalls++
+      gh api "repos/${RepoId}/git/refs/heads/${branchName}" -X DELETE
+      if ($LASTEXITCODE) {
+        LogError "Deletion of branch '$branchName` failed, see command output above"
+        exit $LASTEXITCODE
+      }
+      LogDebug "The branch '$branchName' at commit '$commitUrl' in '$RepoId' has been deleted."
     }
-    Write-Host "The branch '$branchName' at commit '$commitUrl' in '$RepoId' has been deleted."
+  }
+}
+finally {
+
+
+  Write-Host "Number of branches returned from graphql query: $totalBranchesFromQuery"
+  # The $BranchRegex seems to be always set
+  if ($BranchRegex) {
+    Write-Host "Number of branches that didn't match the BranchRegex: $skippedBranchNotMatchRegex"
+  }
+  Write-Host "Number of branches skipped for newer last commit date: $skippedForCommitDate"
+  Write-Host "Number of branches skipped for open PRs: $skippedForOpenPRs"
+  Write-Host "Number of gh api calls to delete branches: $ghBranchDeleteCalls"
+  # The following are only applicable when $CentralRepoId is passed in
+  if ($CentralRepoId) {
+    Write-Host "The following are applicable because CentralRepoId was passed in:"
+    Write-Host "  Number of gh pr view calls: $ghPRViewCalls"
+    Write-Host "  Number of branches skipped due to PR not in the repository: $skippedForPRNotInRepo "
+    Write-Host "  Number of branches skipped due to PR not in the branch name: $skippedForPRNotInBranch"
   }
 }