diff --git a/eng/ci/config/test-groups.json b/eng/ci/config/test-groups.json index a3aa3bdf..539401f6 100644 --- a/eng/ci/config/test-groups.json +++ b/eng/ci/config/test-groups.json @@ -16,14 +16,6 @@ "display": "CosmosDB Tests", "stopEventHub": "false" }, - { - "name": "MySQL", - "group": "mysql", - "files": "test_mysql_functions.py", - "emulators": "mysql", - "display": "MySQL Tests", - "stopEventHub": "false" - }, { "name": "SQL", "group": "sql", @@ -48,14 +40,6 @@ "display": "Durable Task Scheduler Tests", "stopEventHub": "false" }, - { - "name": "Kafka", - "group": "kafka", - "files": "test_kafka_functions.py", - "emulators": "kafka", - "display": "Kafka Tests", - "stopEventHub": "false" - }, { "name": "SignalR", "group": "signalr", @@ -63,14 +47,6 @@ "emulators": "signalr", "display": "SignalR Tests", "stopEventHub": "false" - }, - { - "name": "RabbitMQ", - "group": "rabbitmq", - "files": "test_rabbitmq_functions.py", - "emulators": "rabbitmq", - "display": "RabbitMQ Tests", - "stopEventHub": "true" } ] } \ No newline at end of file diff --git a/eng/ci/scripts/build-core-tools.ps1 b/eng/ci/scripts/build-core-tools.ps1 index 323a6e06..77df39db 100644 --- a/eng/ci/scripts/build-core-tools.ps1 +++ b/eng/ci/scripts/build-core-tools.ps1 @@ -80,16 +80,49 @@ Write-Host "Zip Output Directory: $ZipOutputDir" -ForegroundColor Yellow Write-Host "`nPublishing Azure.Functions.Cli with $Configuration configuration..." -ForegroundColor Yellow Write-Host "Output Directory: $OutputDir" -ForegroundColor Yellow +# Detect target framework from the project file +$projXml = [xml](Get-Content $ProjectPath) +$targetFramework = $projXml.Project.PropertyGroup.TargetFramework | Where-Object { $_ } | Select-Object -First 1 +if (-not $targetFramework) { + $targetFramework = "net8.0" + Write-Host "Could not detect TargetFramework, defaulting to $targetFramework" -ForegroundColor Yellow +} else { + Write-Host "Detected TargetFramework: $targetFramework" -ForegroundColor Green +} + Push-Location $CoreToolsDir try { + # Restore packages first, then publish with --no-restore to avoid redundant downloads + $restoreArgs = @( + "restore", + $ProjectPath, + "/p:TargetFramework=$targetFramework" + ) + + if (-not [string]::IsNullOrEmpty($Runtime)) { + $restoreArgs += "-r", $Runtime + } + + Write-Host "Running: dotnet $($restoreArgs -join ' ')" -ForegroundColor Cyan + & dotnet $restoreArgs 2>&1 | Tee-Object -Variable restoreLogs + + if ($LASTEXITCODE -ne 0) { + Write-Host "`n##[error]dotnet restore failed with exit code $LASTEXITCODE" -ForegroundColor Red + $restoreLogs | ForEach-Object { Write-Host $_ } + throw "Restore failed with exit code $LASTEXITCODE" + } + + Write-Host "✓ Restore completed successfully" -ForegroundColor Green + $publishArgs = @( "publish", $ProjectPath, "-o", $OutputDir, "-c", $Configuration, - "-f", "net8.0", + "-f", $targetFramework, "--self-contained", + "--no-restore", "/p:ZipAfterPublish=true", "/p:ZipArtifactsPath=$ZipOutputDir" ) diff --git a/eng/ci/scripts/build-multi-version-core-tools.ps1 b/eng/ci/scripts/build-multi-version-core-tools.ps1 index 61cad50a..5a1a9399 100644 --- a/eng/ci/scripts/build-multi-version-core-tools.ps1 +++ b/eng/ci/scripts/build-multi-version-core-tools.ps1 @@ -9,11 +9,17 @@ .DESCRIPTION This script: 1. Fetches the latest N host tags from azure-functions-host repository - 2. For each tag version: - - Updates Packages.props with the host version - - Runs validate-worker-versions.ps1 to sync worker versions - - Builds the core tools with that version - 3. Outputs all build artifacts + 2. Resolves each host version to a core-tools tag or origin/main + 3. For each version: + - Checks out the matching core-tools tag or origin/main (no patching needed) + - Applies minimal build compatibility fixes (rollForward, UpdateBuildNumber) + - Builds the core tools + 4. Outputs all build artifacts + + Resolution order per host version: + - Exact core-tools tag match → checkout tag (cleanest, no patching) + - origin/main already has this host version → checkout main (no patching) + - No match → checkout main + run update-core-tools-versions.ps1 (fallback) .PARAMETER Count Number of latest host tags to build (default: 2) @@ -27,7 +33,7 @@ Build configuration - Debug or Release (default: Release) .PARAMETER CloneDir - Directory where the core-tools repository is located + Directory where the core-tools repository is located (must have full git history + tags) .EXAMPLE .\build-multi-version-core-tools.ps1 -Count 2 -Pattern "v4.10" -Configuration Release @@ -65,31 +71,107 @@ if (Test-Path $CloneDir) { Write-Error "Clone directory does not exist: $CloneDir" exit 1 } -$PackagesPropsPath = Join-Path $CloneDir "eng/build/Packages.props" -$UpdateVersionsScript = Join-Path $ScriptDir "update-core-tools-versions.ps1" + $GetLatestTagsScript = Join-Path $ScriptDir "get-latest-host-tags.ps1" +$ResolveTagsScript = Join-Path $ScriptDir "resolve-core-tools-tags.ps1" +$UpdateVersionsScript = Join-Path $ScriptDir "update-core-tools-versions.ps1" +$BuildScript = Join-Path $ScriptDir "build-core-tools.ps1" +$CoreToolsGitHubUrl = "https://github.com/Azure/azure-functions-core-tools.git" +$PackagesPropsPath = Join-Path $CloneDir "eng/build/Packages.props" +$FinalZipDir = Join-Path $CloneDir "artifacts-coretools-zip" Write-Host "`nPaths:" -ForegroundColor Yellow Write-Host " Repo Root: $RepoRoot" Write-Host " Clone Dir: $CloneDir" -Write-Host " Packages.props: $PackagesPropsPath" -Write-Host " Update Versions Script: $UpdateVersionsScript" +Write-Host " Artifacts Output: $FinalZipDir" Write-Host "" -# Verify required files exist -if (-not (Test-Path $PackagesPropsPath)) { - Write-Error "Packages.props not found at: $PackagesPropsPath" - exit 1 +# Verify required scripts exist +foreach ($script in @($GetLatestTagsScript, $ResolveTagsScript, $UpdateVersionsScript, $BuildScript)) { + if (-not (Test-Path $script)) { + Write-Error "Required script not found: $script" + exit 1 + } } -if (-not (Test-Path $UpdateVersionsScript)) { - Write-Error "update-core-tools-versions.ps1 not found at: $UpdateVersionsScript" - exit 1 +# Create artifacts output directory +if (-not (Test-Path $FinalZipDir)) { + New-Item -ItemType Directory -Path $FinalZipDir -Force | Out-Null } -if (-not (Test-Path $GetLatestTagsScript)) { - Write-Error "get-latest-host-tags.ps1 not found at: $GetLatestTagsScript" - exit 1 +# Applies minimal build compatibility fixes after checkout. +# These are universal fixes needed regardless of checkout source (tag, main, or fallback). +function Apply-PostCheckoutFixes { + param( + [string]$RepoDir, + [string]$HostVersion + ) + + Write-Host " Applying post-checkout compatibility fixes..." -ForegroundColor Gray + + # Fix 1: Sync global.json SDK version from the host repo, then adjust to installed SDKs + # The host's global.json is the source of truth for SDK expectations. + $globalJsonPath = Join-Path $RepoDir "global.json" + if (Test-Path $globalJsonPath) { + $globalJson = Get-Content $globalJsonPath -Raw | ConvertFrom-Json + $oldSdkVersion = $globalJson.sdk.version + $oldRollForward = $globalJson.sdk.rollForward + + # Step 1a: Fetch SDK version from the host repo's global.json + $hostGlobalJsonUri = "https://raw.githubusercontent.com/Azure/azure-functions-host/refs/tags/v$HostVersion/global.json" + try { + $hostGlobalJsonContent = (Invoke-WebRequest -Uri $hostGlobalJsonUri -Headers @{"User-Agent" = "azure-functions-extension-bundles"} -ErrorAction Stop).Content + $hostGlobalJson = $hostGlobalJsonContent | ConvertFrom-Json + $hostSdkVersion = $hostGlobalJson.sdk.version + + if ($oldSdkVersion -ne $hostSdkVersion) { + $globalJson.sdk.version = $hostSdkVersion + Write-Host " ✓ Synced SDK version from host: $oldSdkVersion -> $hostSdkVersion" -ForegroundColor Green + } else { + Write-Host " SDK version already matches host: $oldSdkVersion" -ForegroundColor Gray + } + } catch { + Write-Host " ⚠ Could not fetch host global.json, keeping core-tools version: $oldSdkVersion" -ForegroundColor Yellow + } + + # Step 1b: Set rollForward to latestFeature (stay within major, allow patch flexibility) + $globalJson.sdk.rollForward = "latestFeature" + + # Step 1c: Adjust SDK version to closest installed match if exact version isn't available + $requestedVersion = $globalJson.sdk.version + $installedSdks = (dotnet --list-sdks 2>$null) | ForEach-Object { ($_ -split '\s')[0] } + + if ($installedSdks -and $requestedVersion) { + $reqParts = $requestedVersion -split '\.' + $reqMajorMinor = "$($reqParts[0]).$($reqParts[1])" + + # Find installed SDKs in the same major.minor + $matchingSdks = $installedSdks | Where-Object { $_.StartsWith("$reqMajorMinor.") } | Sort-Object { [version]$_ } -Descending + + if ($matchingSdks -and ($matchingSdks -notcontains $requestedVersion)) { + $bestMatch = $matchingSdks | Select-Object -First 1 + $globalJson.sdk.version = $bestMatch + Write-Host " ✓ Adjusted SDK to installed: $requestedVersion -> $bestMatch" -ForegroundColor Green + } + } + + $globalJson | ConvertTo-Json -Depth 10 | Set-Content $globalJsonPath + if ($oldRollForward -ne "latestFeature") { + Write-Host " ✓ Set rollForward to latestFeature (was: $oldRollForward)" -ForegroundColor Green + } + } + + # Fix 2: Disable UpdateBuildNumber to prevent build from updating Azure DevOps build number + $versionPropsPath = Join-Path $RepoDir "src/Cli/func/Directory.Version.props" + if (Test-Path $versionPropsPath) { + [xml]$versionPropsXml = Get-Content $versionPropsPath + $node = Select-Xml -Xml $versionPropsXml -XPath "//UpdateBuildNumber" | Select-Object -ExpandProperty Node + if ($node -and $node.'#text' -ne 'false') { + $node.'#text' = 'false' + $versionPropsXml.Save($versionPropsPath) + Write-Host " ✓ Set UpdateBuildNumber to false" -ForegroundColor Green + } + } } # Step 1: Get latest host tags @@ -106,148 +188,144 @@ $latestTags | ForEach-Object { Write-Host " - $($_.Tag) -> $($_.VersionNoPrefix)" -ForegroundColor White } -# Backup all files that update-core-tools-versions.ps1 modifies -# This ensures clean state for each version build -$BackupFiles = @{ - PackagesProps = @{ - Path = $PackagesPropsPath - Backup = "$PackagesPropsPath.backup" - } - GlobalJson = @{ - Path = Join-Path $CloneDir "global.json" - Backup = Join-Path $CloneDir "global.json.backup" - } - StartupCs = @{ - Path = Join-Path $CloneDir "src\Cli\func\Actions\HostActions\Startup.cs" - Backup = Join-Path $CloneDir "src\Cli\func\Actions\HostActions\Startup.cs.backup" - } - DirectoryVersionProps = @{ - Path = Join-Path $CloneDir "src\Cli\func\Directory.Version.props" - Backup = Join-Path $CloneDir "src\Cli\func\Directory.Version.props.backup" - } -} +# Step 2: Resolve host versions to core-tools refs +Write-Host "`nStep 2: Resolving core-tools refs for each host version..." -ForegroundColor Cyan +$hostVersions = $latestTags | ForEach-Object { $_.VersionNoPrefix } +$refMap = & $ResolveTagsScript -HostVersions $hostVersions -CoreToolsDir $CloneDir -Write-Host "`nBacking up files that will be modified..." -ForegroundColor Gray -foreach ($fileKey in $BackupFiles.Keys) { - $fileInfo = $BackupFiles[$fileKey] - if (Test-Path $fileInfo.Path) { - Copy-Item $fileInfo.Path $fileInfo.Backup -Force - Write-Host " ✓ Backed up: $($fileInfo.Path)" -ForegroundColor Green - } else { - Write-Host " - Skipped (not found): $($fileInfo.Path)" -ForegroundColor Gray - } +if (-not $refMap) { + Write-Error "Failed to resolve core-tools tags" + exit 1 } # Array to store build results $buildResults = @() try { - # Step 2: Build each version + # Step 3: Build each version $versionIndex = 1 foreach ($tagInfo in $latestTags) { $hostVersion = $tagInfo.VersionNoPrefix + $refInfo = $refMap[$hostVersion] Write-Host "`n===========================================================" -ForegroundColor Cyan Write-Host "Building Version $versionIndex of $($latestTags.Count): $hostVersion" -ForegroundColor Cyan + Write-Host " Resolution: $($refInfo.DisplayRef)" -ForegroundColor White Write-Host "===========================================================" -ForegroundColor Cyan - # Restore all modified files to original state before each build - Write-Host "`nRestoring files to original state..." -ForegroundColor Gray - foreach ($fileKey in $BackupFiles.Keys) { - $fileInfo = $BackupFiles[$fileKey] - if (Test-Path $fileInfo.Backup) { - Copy-Item $fileInfo.Backup $fileInfo.Path -Force - Write-Host " ✓ Restored: $($fileInfo.Path)" -ForegroundColor Green - } + # Step 3a: Reset repo to clean state + Write-Host "`nStep 3.$versionIndex.a: Resetting repo to clean state..." -ForegroundColor Yellow + Push-Location $CloneDir + try { + $null = git reset --hard 2>&1 + # Clean untracked files but preserve the final artifacts directory + $null = git clean -fdx -e "artifacts-coretools-zip" 2>&1 + Write-Host " ✓ Repo reset to clean state" -ForegroundColor Green + } finally { + Pop-Location } - - # Step 2a: Update host and worker versions in Packages.props - Write-Host "`nStep 2.$versionIndex.a: Updating host and worker versions to $hostVersion" -ForegroundColor Yellow + # Step 3b: Checkout the resolved ref + Write-Host "`nStep 3.$versionIndex.b: Checking out $($refInfo.DisplayRef)..." -ForegroundColor Yellow + Push-Location $CloneDir try { - & $UpdateVersionsScript -HostVersion $hostVersion -PackagesPropsPath $PackagesPropsPath - - if ($LASTEXITCODE -ne 0) { - throw "update-core-tools-versions.ps1 failed with exit code $LASTEXITCODE" + if ($refInfo.Type -eq "tag") { + # Shallow clone may not have the tag's commit — fetch from GitHub + Write-Host " Fetching tag $($refInfo.Ref) from GitHub..." -ForegroundColor Gray + $fetchOutput = git fetch $CoreToolsGitHubUrl tag $refInfo.Ref --no-tags --depth=1 2>&1 + if ($LASTEXITCODE -ne 0) { throw "git fetch tag failed: $fetchOutput" } + $checkoutOutput = git checkout $refInfo.Ref 2>&1 + if ($LASTEXITCODE -ne 0) { throw "git checkout failed: $checkoutOutput" } + } else { + # main or fallback — fetch latest main from GitHub + Write-Host " Fetching main from GitHub..." -ForegroundColor Gray + $fetchOutput = git fetch $CoreToolsGitHubUrl main --depth=1 2>&1 + if ($LASTEXITCODE -ne 0) { throw "git fetch main failed: $fetchOutput" } + $checkoutOutput = git checkout FETCH_HEAD 2>&1 + if ($LASTEXITCODE -ne 0) { throw "git checkout FETCH_HEAD failed: $checkoutOutput" } } - - Write-Host " ✓ Host and worker versions updated successfully" -ForegroundColor Green + Write-Host " ✓ Checked out $($refInfo.DisplayRef)" -ForegroundColor Green + } finally { + Pop-Location } - catch { - throw "Failed to update versions: $_" + + # Step 3c: Apply version patching if this is a fallback (no exact match) + if ($refInfo.Type -eq "fallback") { + Write-Host "`nStep 3.$versionIndex.c: Applying version patching (fallback)..." -ForegroundColor Yellow + try { + & $UpdateVersionsScript -HostVersion $hostVersion -PackagesPropsPath $PackagesPropsPath + if ($LASTEXITCODE -ne 0) { + throw "update-core-tools-versions.ps1 failed with exit code $LASTEXITCODE" + } + Write-Host " ✓ Version patching applied" -ForegroundColor Green + } catch { + throw "Failed to apply version patching: $_" + } + } else { + Write-Host "`nStep 3.$versionIndex.c: No patching needed ($($refInfo.Type) match)" -ForegroundColor Green } - # Step 2b: Build core tools - Write-Host "`nStep 2.$versionIndex.b: Building Core Tools for version $hostVersion..." -ForegroundColor Yellow + # Step 3d: Apply minimal post-checkout fixes (all paths) + Apply-PostCheckoutFixes -RepoDir $CloneDir -HostVersion $hostVersion + + # Step 3e: Build core tools + Write-Host "`nStep 3.$versionIndex.e: Building Core Tools for version $hostVersion..." -ForegroundColor Yellow - $buildScript = Join-Path $ScriptDir "build-core-tools.ps1" - # Use version-specific output directory to prevent overwriting + # Use a temp zip dir inside the repo (will be cleaned by git clean next iteration) $versionZipDir = "artifacts-coretools-zip-$hostVersion" - $buildOutput = & $buildScript -Configuration $Configuration -CoreToolsDir $CloneDir -ZipOutputDir $versionZipDir + $buildOutput = & $BuildScript -Configuration $Configuration -CoreToolsDir $CloneDir -ZipOutputDir $versionZipDir if ($LASTEXITCODE -ne 0) { throw "Core Tools build failed with exit code $LASTEXITCODE" } - # Step 2c: Copy and rename zip files to consolidated artifacts directory - Write-Host "`nStep 2.$versionIndex.c: Consolidating artifacts for host version $hostVersion..." -ForegroundColor Yellow + # Step 3f: Move artifacts to output directory (outside repo tree) + Write-Host "`nStep 3.$versionIndex.f: Moving artifacts..." -ForegroundColor Yellow $tempZipDir = Join-Path $CloneDir $versionZipDir - $finalZipDir = Join-Path $CloneDir "artifacts-coretools-zip" - - # Create final directory if it doesn't exist - if (-not (Test-Path $finalZipDir)) { - New-Item -ItemType Directory -Path $finalZipDir -Force | Out-Null - } - if (Test-Path $tempZipDir) { - # Get only the first zip file to avoid overwrites if multiple exist $zipFile = Get-ChildItem -Path $tempZipDir -Filter "*.zip" | Select-Object -First 1 - if ($zipFile) { - $oldName = $zipFile.Name - # Use iteration-prefixed naming: {iteration}-cli-host-{version}.zip $newName = "$versionIndex-cli-host-$hostVersion.zip" - $destPath = Join-Path $finalZipDir $newName - + $destPath = Join-Path $FinalZipDir $newName Copy-Item -Path $zipFile.FullName -Destination $destPath -Force - Write-Host " Copied and renamed: $oldName -> $newName" -ForegroundColor Green + Write-Host " ✓ $($zipFile.Name) -> $newName" -ForegroundColor Green } else { Write-Warning " No zip files found in $tempZipDir" } - - # Clean up version-specific directory - Remove-Item -Path $tempZipDir -Recurse -Force } - # Step 2d: Clean up intermediate build artifacts to save disk space - Write-Host "`nStep 2.$versionIndex.d: Cleaning up intermediate build artifacts..." -ForegroundColor Yellow + # Step 3g: Clean up to save disk space + Write-Host "`nStep 3.$versionIndex.g: Cleaning up..." -ForegroundColor Yellow $artifactsDir = Join-Path $CloneDir "artifacts" if (Test-Path $artifactsDir) { Remove-Item -Path $artifactsDir -Recurse -Force - Write-Host " ✓ Removed artifacts directory" -ForegroundColor Green } - # Clean up obj/bin directories in source + # Clean obj/bin $srcDir = Join-Path $CloneDir "src" if (Test-Path $srcDir) { Get-ChildItem -Path $srcDir -Include "obj","bin" -Recurse -Directory -Force | ForEach-Object { Remove-Item -Path $_.FullName -Recurse -Force -ErrorAction SilentlyContinue } - Write-Host " ✓ Cleaned obj/bin directories" -ForegroundColor Green } + + # Clear NuGet cache between builds to reclaim disk space + Write-Host " Clearing NuGet cache..." -ForegroundColor Gray + & dotnet nuget locals all --clear 2>$null + Write-Host " ✓ Cleanup complete" -ForegroundColor Green # Store result $buildResults += [PSCustomObject]@{ HostVersion = $hostVersion Tag = $tagInfo.Tag - BuildOutput = $buildOutput + CoreToolsRef = $refInfo.DisplayRef + RefType = $refInfo.Type Success = $true } Write-Host "`n ✓ Successfully built Core Tools for version $hostVersion" -ForegroundColor Green - Write-Host " Build Output: $buildOutput" -ForegroundColor White $versionIndex++ } @@ -259,41 +337,28 @@ try { Write-Host "Successfully built $($buildResults.Count) version(s):" -ForegroundColor Green $buildResults | ForEach-Object { - Write-Host " ✓ $($_.Tag) ($($_.HostVersion))" -ForegroundColor Green - Write-Host " Output: $($_.BuildOutput)" -ForegroundColor White + $icon = if ($_.RefType -eq "fallback") { "⚠" } else { "✓" } + Write-Host " $icon $($_.Tag) ($($_.HostVersion)) via $($_.CoreToolsRef)" -ForegroundColor $(if ($_.RefType -eq "fallback") { "Yellow" } else { "Green" }) } - # Verify artifacts directory - $finalZipDir = Join-Path $CloneDir "artifacts-coretools-zip" - Write-Host "`nVerifying artifacts in: $finalZipDir" -ForegroundColor Cyan - if (Test-Path $finalZipDir) { - $zipFiles = Get-ChildItem -Path $finalZipDir -Filter "*.zip" + # Verify artifacts + Write-Host "`nVerifying artifacts in: $FinalZipDir" -ForegroundColor Cyan + if (Test-Path $FinalZipDir) { + $zipFiles = Get-ChildItem -Path $FinalZipDir -Filter "*.zip" Write-Host "Found $($zipFiles.Count) zip file(s):" -ForegroundColor Green $zipFiles | ForEach-Object { $sizeKB = [math]::Round($_.Length / 1KB, 2) Write-Host " - $($_.Name) (${sizeKB} KB)" -ForegroundColor White } } else { - Write-Host "##[error]Artifacts directory not found: $finalZipDir" -ForegroundColor Red + Write-Host "##[error]Artifacts directory not found: $FinalZipDir" -ForegroundColor Red } Write-Host "`n===========================================================" -ForegroundColor Cyan - # Return build results return $buildResults } catch { Write-Error "Build failed: $_" exit 1 -} finally { - # Restore all original files and clean up backups - Write-Host "`nRestoring original files and cleaning up backups..." -ForegroundColor Gray - foreach ($fileKey in $BackupFiles.Keys) { - $fileInfo = $BackupFiles[$fileKey] - if (Test-Path $fileInfo.Backup) { - Copy-Item $fileInfo.Backup $fileInfo.Path -Force - Remove-Item $fileInfo.Backup -Force - Write-Host " ✓ Restored and cleaned up: $($fileInfo.Path)" -ForegroundColor Green - } - } } diff --git a/eng/ci/scripts/get-latest-host-tags.ps1 b/eng/ci/scripts/get-latest-host-tags.ps1 index 3ff97c0a..c2291297 100644 --- a/eng/ci/scripts/get-latest-host-tags.ps1 +++ b/eng/ci/scripts/get-latest-host-tags.ps1 @@ -90,6 +90,20 @@ $parsedTags = $tags | ForEach-Object { } } | Where-Object { $_ -ne $null } +# Exclude host versions >= 4.1049 which require dotnet 10. +# TODO: Remove this filter once core tools complete migration to dotnet 10. +$parsedTags = $parsedTags | Where-Object { + $versionParts = $_.VersionNoPrefix -split '\.' + [int]$versionParts[1] -lt 1049 +} + +if (-not $parsedTags) { + Write-Error "No tags found after filtering for versions < 4.1049" + exit 1 +} + +Write-Host "Tags after filtering (< 4.1049): $($parsedTags.Count)" -ForegroundColor Green + # Group by middle version and get the highest patch from each group $groupedTags = $parsedTags | Group-Object MiddleVersion | diff --git a/eng/ci/scripts/resolve-core-tools-tags.ps1 b/eng/ci/scripts/resolve-core-tools-tags.ps1 new file mode 100644 index 00000000..51bb3af8 --- /dev/null +++ b/eng/ci/scripts/resolve-core-tools-tags.ps1 @@ -0,0 +1,146 @@ +#!/usr/bin/env pwsh +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +<# +.SYNOPSIS + Maps host versions to core-tools tags by inspecting Packages.props in each tag. + +.DESCRIPTION + For each requested host version, this script finds the core-tools release tag + that ships with that exact host version. It uses git ls-remote to list tags + and fetches Packages.props via GitHub raw content URL (works with shallow clones). + + Resolution order: + 1. origin/main already has the requested host version + 2. Exact match in a release tag (e.g., tag 4.9.0 ships host 4.1047.100) + 3. No match — caller should fall back to patching main + +.PARAMETER HostVersions + Array of host versions to resolve (e.g., @("4.1047.100", "4.1048.100")) + +.PARAMETER CoreToolsDir + Path to the core-tools git repository + +.EXAMPLE + .\resolve-core-tools-tags.ps1 -HostVersions @("4.1047.100", "4.1046.100") -CoreToolsDir "path/to/core-tools" + Returns hashtable: @{ "4.1047.100" = @{ Ref = "4.9.0"; Type = "tag" }; "4.1046.100" = @{ Ref = "4.8.0"; Type = "tag" } } +#> + +param( + [Parameter(Mandatory=$true)] + [string[]]$HostVersions, + + [Parameter(Mandatory=$true)] + [string]$CoreToolsDir +) + +$ErrorActionPreference = "Stop" + +Write-Host "===========================================================" -ForegroundColor Cyan +Write-Host "Resolving Core Tools Tags for Host Versions" -ForegroundColor Cyan +Write-Host "===========================================================" -ForegroundColor Cyan +Write-Host "Host versions to resolve: $($HostVersions -join ', ')" -ForegroundColor Yellow + +$GitHubHeaders = @{ "User-Agent" = "azure-functions-extension-bundles-emulator-tests" } + +# Helper: extract WebHost version from Packages.props content +function Get-HostVersionFromProps { + param([string]$Content) + if ($Content -match 'Include="Microsoft\.Azure\.WebJobs\.Script\.WebHost"\s+Version="([^"]+)"') { + return $matches[1] + } + return $null +} + +# Build set of versions we still need to resolve +$unresolved = @{} +foreach ($hv in $HostVersions) { + $unresolved[$hv] = $true +} + +# Result hashtable: hostVersion -> @{ Ref = "tag/ref"; Type = "tag"|"main"|"fallback" } +$result = @{} + +Push-Location $CoreToolsDir +try { + # Step 1: Check main branch via GitHub raw URL (works with shallow clones) + Write-Host "`nChecking main branch..." -ForegroundColor Yellow + $mainPropsUrl = "https://raw.githubusercontent.com/Azure/azure-functions-core-tools/refs/heads/main/eng/build/Packages.props" + try { + $mainPropsResponse = Invoke-WebRequest -Uri $mainPropsUrl -Headers $GitHubHeaders -ErrorAction Stop -TimeoutSec 10 + $mainHostVersion = Get-HostVersionFromProps -Content $mainPropsResponse.Content + Write-Host " main has host version: $mainHostVersion" -ForegroundColor Gray + + if ($mainHostVersion -and $unresolved.ContainsKey($mainHostVersion)) { + $result[$mainHostVersion] = @{ Ref = "main"; Type = "main"; DisplayRef = "main (host $mainHostVersion)" } + $unresolved.Remove($mainHostVersion) + Write-Host " ✓ Matched main for host $mainHostVersion" -ForegroundColor Green + } + } catch { + Write-Host " ⚠ Could not check main branch: $_" -ForegroundColor Yellow + } + + if ($unresolved.Count -eq 0) { + Write-Host "`nAll host versions resolved!" -ForegroundColor Green + return $result + } + + # Step 2: List core-tools tags via git ls-remote (same pattern as get-latest-host-tags.ps1) + Write-Host "`nScanning core-tools tags via GitHub..." -ForegroundColor Yellow + $coreToolsRepoUrl = "https://github.com/Azure/azure-functions-core-tools.git" + $remoteTags = git ls-remote --tags $coreToolsRepoUrl 2>&1 | + Where-Object { $_ -match 'refs/tags/(\d+\.\d+\.\d+)(\^\{\})?$' } | + ForEach-Object { + if ($_ -match 'refs/tags/(\d+\.\d+\.\d+)(\^\{\})?$') { $matches[1] } + } | Select-Object -Unique | + Where-Object { $_.StartsWith("4.") } | + Sort-Object { [version]$_ } -Descending + + if (-not $remoteTags) { + Write-Warning "No core-tools tags found via git ls-remote" + } else { + Write-Host " Found $($remoteTags.Count) core-tools tag(s), scanning newest first..." -ForegroundColor Gray + + foreach ($tag in $remoteTags) { + if ($unresolved.Count -eq 0) { break } + + # Fetch Packages.props from GitHub raw content (no local checkout needed) + $rawUrl = "https://raw.githubusercontent.com/Azure/azure-functions-core-tools/refs/tags/$tag/eng/build/Packages.props" + try { + $response = Invoke-WebRequest -Uri $rawUrl -Headers $GitHubHeaders -ErrorAction Stop -TimeoutSec 10 + $tagHostVersion = Get-HostVersionFromProps -Content $response.Content + } catch { + Write-Host " Skipping tag $tag (fetch failed)" -ForegroundColor Gray + continue + } + + if (-not $tagHostVersion) { continue } + + if ($unresolved.ContainsKey($tagHostVersion)) { + $result[$tagHostVersion] = @{ Ref = $tag; Type = "tag"; DisplayRef = "tag $tag (host $tagHostVersion)" } + $unresolved.Remove($tagHostVersion) + Write-Host " ✓ Tag $tag -> host $tagHostVersion" -ForegroundColor Green + } + } + } + + # Step 3: Anything still unresolved will use main + patching + foreach ($hv in @($unresolved.Keys)) { + $result[$hv] = @{ Ref = "main"; Type = "fallback"; DisplayRef = "main + patching (host $hv)" } + Write-Host " ⚠ No exact match for host $hv — will use main + patching" -ForegroundColor Yellow + } + + # Summary + Write-Host "`n--- Resolution Summary ---" -ForegroundColor Cyan + foreach ($hv in $HostVersions) { + $info = $result[$hv] + $icon = if ($info.Type -eq "fallback") { "⚠" } else { "✓" } + Write-Host " $icon Host $hv -> $($info.DisplayRef)" -ForegroundColor $(if ($info.Type -eq "fallback") { "Yellow" } else { "Green" }) + } + + return $result + +} finally { + Pop-Location +} diff --git a/eng/ci/scripts/update-core-tools-versions.ps1 b/eng/ci/scripts/update-core-tools-versions.ps1 index 1d78f027..8cdfc66d 100644 --- a/eng/ci/scripts/update-core-tools-versions.ps1 +++ b/eng/ci/scripts/update-core-tools-versions.ps1 @@ -64,7 +64,7 @@ if (Test-Path $GlobalJsonPath) { if ($localSdkVersion -ne $hostSdkVersion) { # Update SDK version from host repo $localGlobalJson.sdk.version = $hostSdkVersion - $localGlobalJson.sdk.rollForward = "latestMajor" + $localGlobalJson.sdk.rollForward = "latestFeature" $localGlobalJson | ConvertTo-Json -Depth 10 | Set-Content $GlobalJsonPath Write-Host " ✓ Updated global.json SDK: $localSdkVersion -> $hostSdkVersion (from host repo)" -ForegroundColor Green } else { @@ -182,6 +182,7 @@ $workers = @{ "Microsoft.Azure.Functions.PowerShellWorker.PS7.0" = "eng/build/Workers.Powershell.props" "Microsoft.Azure.Functions.PowerShellWorker.PS7.2" = "eng/build/Workers.Powershell.props" "Microsoft.Azure.Functions.PowerShellWorker.PS7.4" = "eng/build/Workers.Powershell.props" + "Microsoft.Azure.Functions.PowerShellWorker.PS7.6" = "eng/build/Workers.Powershell.props" } # Fix until core tool can sync with new host changes @@ -333,52 +334,9 @@ Write-Host "`nSaving updated Packages.props..." -ForegroundColor Yellow $packagesXml.Save($PackagesPropsPath) Write-Host "✓ Successfully updated Packages.props" -ForegroundColor Green -# Get core tools root path for subsequent file modifications until core tools adopt the new host changes in v4.1047.100 +# Get core tools root path for subsequent file modifications $coreToolsRoot = Split-Path (Split-Path (Split-Path $PackagesPropsPath -Parent) -Parent) -Parent -# Update Startup.cs to remove deprecated IApplicationLifetime usage (for host version >= 4.1047.100) -Write-Host "`nChecking for Startup.cs updates..." -ForegroundColor Yellow - -if ($shouldApplyHostFixes) { - $startupPath = Join-Path $coreToolsRoot "src\Cli\func\Actions\HostActions\Startup.cs" - - if (Test-Path $startupPath) { - $startupContent = Get-Content $startupPath -Raw - - # Define the old code pattern to replace - $oldCode = @" - } -#pragma warning disable CS0618 // IApplicationLifetime is obsolete - IApplicationLifetime applicationLifetime = app.ApplicationServices - .GetRequiredService(); - - app.UseWebJobsScriptHost(applicationLifetime); -#pragma warning restore CS0618 // Type is obsolete - } -"@ - - # Define the new simplified code - $newCode = @" - } - - app.UseWebJobsScriptHost(); - } -"@ - - if ($startupContent.Contains($oldCode)) { - $startupContent = $startupContent.Replace($oldCode, $newCode) - Set-Content -Path $startupPath -Value $startupContent -NoNewline - Write-Host " ✓ Updated Startup.cs - removed deprecated IApplicationLifetime usage" -ForegroundColor Green - } else { - Write-Host " Startup.cs already updated or pattern not found (no change needed)" -ForegroundColor Gray - } - } else { - Write-Warning " Startup.cs not found at: $startupPath" - } -} else { - Write-Host " Skipping Startup.cs update (host version $HostVersion < 4.1047.100)" -ForegroundColor Gray -} - # Disable UpdateBuildNumber in Directory.Version.props to prevent build output from updating Azure DevOps build number Write-Host "`nDisabling UpdateBuildNumber in Directory.Version.props..." -ForegroundColor Yellow $versionPropsPath = Join-Path $coreToolsRoot "src\Cli\func\Directory.Version.props" diff --git a/eng/ci/templates/jobs/emulator-tests.yml b/eng/ci/templates/jobs/emulator-tests.yml index d6730b6a..0039be8a 100644 --- a/eng/ci/templates/jobs/emulator-tests.yml +++ b/eng/ci/templates/jobs/emulator-tests.yml @@ -40,7 +40,7 @@ jobs: - script: | echo "Disk space before build:" df -h - displayName: 'Check Initial Disk Space' + displayName: 'Check Disk Space Before Build' - task: NuGetAuthenticate@1 displayName: 'NuGet Authenticate' @@ -56,6 +56,10 @@ jobs: - script: | echo "Cleaning up build artifacts to free disk space..." + # Remove .git directory (no longer needed after build) + echo "Removing core-tools .git directory..." + rm -rf $(Agent.BuildDirectory)/core-tools/.git 2>/dev/null || true + # Remove source code (keep only artifacts) echo "Removing Core Tools source files..." find $(Agent.BuildDirectory)/core-tools -type f -name "*.cs" -delete 2>/dev/null || true @@ -66,10 +70,6 @@ jobs: echo "Clearing NuGet cache..." rm -rf ~/.nuget/packages/* 2>/dev/null || true - # Remove git history from core-tools to save space - echo "Removing git history..." - rm -rf $(Agent.BuildDirectory)/core-tools/.git 2>/dev/null || true - echo "Disk space after cleanup:" df -h displayName: 'Cleanup After Core Tools Build'