Merge pull request #1027 from dotnet/abhitejjohn-eval-correctness-inf… #2698
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: codeowners-folder-validation | |
| on: | |
| pull_request: | |
| push: | |
| branches: [main] | |
| paths: | |
| - 'plugins/**' | |
| - 'tests/**' | |
| - '.github/CODEOWNERS' | |
| - '.github/workflows/codeowners-folder-validation.yml' | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| issues: write | |
| jobs: | |
| validate-codeowners: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 | |
| with: | |
| persist-credentials: false | |
| - name: Find missing CODEOWNERS folder entries | |
| id: audit | |
| shell: pwsh | |
| run: | | |
| $codeownersPath = '.github/CODEOWNERS' | |
| if (-not (Test-Path $codeownersPath)) { | |
| Write-Error "Missing $codeownersPath" | |
| exit 1 | |
| } | |
| # Collect CODEOWNERS path tokens that represent concrete directory scopes. | |
| # Wildcard tokens are ignored for this validation to keep checks folder-specific. | |
| $codeownerEntries = New-Object 'System.Collections.Generic.List[PSCustomObject]' | |
| $ownedPaths = New-Object 'System.Collections.Generic.List[string]' | |
| $lines = Get-Content -Path $codeownersPath | |
| foreach ($line in $lines) { | |
| $trimmed = $line.Trim() | |
| if ([string]::IsNullOrWhiteSpace($trimmed) -or $trimmed.StartsWith('#')) { | |
| continue | |
| } | |
| $tokens = $trimmed -split '\s+' | |
| $pathToken = $tokens[0] | |
| if ($pathToken -match '[*?\[]') { | |
| continue | |
| } | |
| if (-not $pathToken.StartsWith('/')) { | |
| $pathToken = "/$pathToken" | |
| } | |
| if (-not $pathToken.EndsWith('/')) { | |
| $pathToken = "$pathToken/" | |
| } | |
| $owners = @($tokens | Select-Object -Skip 1 | Where-Object { $_ -match '^@' }) | |
| $codeownerEntries.Add([PSCustomObject]@{ | |
| Path = $pathToken | |
| Owners = $owners | |
| }) | |
| $ownedPaths.Add($pathToken) | |
| } | |
| $expectedPaths = New-Object 'System.Collections.Generic.HashSet[string]' ([System.StringComparer]::OrdinalIgnoreCase) | |
| if (Test-Path 'plugins') { | |
| $pluginDirs = Get-ChildItem -Path 'plugins' -Directory | |
| foreach ($pluginDir in $pluginDirs) { | |
| $skillsRoot = Join-Path $pluginDir.FullName 'skills' | |
| if (-not (Test-Path $skillsRoot)) { | |
| continue | |
| } | |
| $skillDirs = Get-ChildItem -Path $skillsRoot -Directory | |
| foreach ($skillDir in $skillDirs) { | |
| [void]$expectedPaths.Add("/plugins/$($pluginDir.Name)/skills/$($skillDir.Name)/") | |
| } | |
| } | |
| } | |
| if (Test-Path 'tests') { | |
| $testPluginDirs = Get-ChildItem -Path 'tests' -Directory | |
| foreach ($testPluginDir in $testPluginDirs) { | |
| $testSkillDirs = Get-ChildItem -Path $testPluginDir.FullName -Directory | |
| foreach ($testSkillDir in $testSkillDirs) { | |
| [void]$expectedPaths.Add("/tests/$($testPluginDir.Name)/$($testSkillDir.Name)/") | |
| } | |
| } | |
| } | |
| function Test-IsCovered([string]$ExpectedPath, [System.Collections.Generic.List[string]]$Scopes) { | |
| foreach ($scope in $Scopes) { | |
| if ($ExpectedPath.StartsWith($scope, [System.StringComparison]::OrdinalIgnoreCase)) { | |
| return $true | |
| } | |
| } | |
| return $false | |
| } | |
| # Return the last matching CODEOWNERS entry (last-match-wins semantics). | |
| function Get-EffectiveEntry([string]$ExpectedPath, [System.Collections.Generic.List[PSCustomObject]]$Entries) { | |
| $lastMatch = $null | |
| foreach ($entry in $Entries) { | |
| if ($ExpectedPath.StartsWith($entry.Path, [System.StringComparison]::OrdinalIgnoreCase)) { | |
| $lastMatch = $entry | |
| } | |
| } | |
| return $lastMatch | |
| } | |
| # Require at least one team or at least two individuals. | |
| function Test-SufficientOwners([string[]]$Owners) { | |
| $teams = @($Owners | Where-Object { $_ -match '/' }) | |
| if ($teams.Count -ge 1) { return $true } | |
| $individuals = @($Owners | Where-Object { $_ -notmatch '/' }) | |
| if ($individuals.Count -ge 2) { return $true } | |
| return $false | |
| } | |
| # --- Check 1: missing CODEOWNERS entries --- | |
| $missing = @( | |
| $expectedPaths | | |
| Where-Object { -not (Test-IsCovered $_ $ownedPaths) } | | |
| Sort-Object | |
| ) | |
| # --- Check 2: insufficient owners (only for covered paths) --- | |
| $insufficientOwners = @( | |
| $expectedPaths | | |
| Where-Object { Test-IsCovered $_ $ownedPaths } | | |
| ForEach-Object { | |
| $entry = Get-EffectiveEntry $_ $codeownerEntries | |
| if ($entry -and -not (Test-SufficientOwners $entry.Owners)) { | |
| [PSCustomObject]@{ | |
| path = $_ | |
| owners = $entry.Owners -join ' ' | |
| } | |
| } | |
| } | | |
| Sort-Object -Property path | |
| ) | |
| # --- Outputs --- | |
| $missingJson = ConvertTo-Json -InputObject @($missing) -Compress | |
| if (-not $missingJson) { $missingJson = '[]' } | |
| $insufficientJson = ConvertTo-Json -InputObject @($insufficientOwners) -Compress -Depth 3 | |
| if (-not $insufficientJson) { $insufficientJson = '[]' } | |
| $hasMissing = $missing.Count -gt 0 | |
| $hasInsufficient = $insufficientOwners.Count -gt 0 | |
| if ($hasMissing) { | |
| Write-Host "Missing CODEOWNERS entries:" | |
| $missing | ForEach-Object { Write-Host " - $_" } | |
| } | |
| if ($hasInsufficient) { | |
| Write-Host "Insufficient owners (need 2+ individuals or 1+ team):" | |
| $insufficientOwners | ForEach-Object { Write-Host " - $($_.path) (current: $($_.owners))" } | |
| } | |
| if (-not $hasMissing -and -not $hasInsufficient) { | |
| Write-Host 'All CODEOWNERS entries are present and have sufficient owners.' | |
| } | |
| "has_missing=$($hasMissing.ToString().ToLower())" >> $env:GITHUB_OUTPUT | |
| "missing_json=$missingJson" >> $env:GITHUB_OUTPUT | |
| "has_insufficient_owners=$($hasInsufficient.ToString().ToLower())" >> $env:GITHUB_OUTPUT | |
| "insufficient_owners_json=$insufficientJson" >> $env:GITHUB_OUTPUT | |
| - name: Create or update issue for validation failures | |
| if: (steps.audit.outputs.has_missing == 'true' || steps.audit.outputs.has_insufficient_owners == 'true') && github.event_name != 'pull_request' | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 | |
| env: | |
| MISSING_JSON: ${{ steps.audit.outputs.missing_json }} | |
| INSUFFICIENT_OWNERS_JSON: ${{ steps.audit.outputs.insufficient_owners_json }} | |
| with: | |
| script: | | |
| const missing = JSON.parse(process.env.MISSING_JSON || '[]'); | |
| const insufficientOwners = JSON.parse(process.env.INSUFFICIENT_OWNERS_JSON || '[]'); | |
| if ((!Array.isArray(missing) || missing.length === 0) && | |
| (!Array.isArray(insufficientOwners) || insufficientOwners.length === 0)) { | |
| core.info('No issues to report.'); | |
| return; | |
| } | |
| const title = 'CODEOWNERS validation failures for skill/test folders'; | |
| const marker = '<!-- codeowners-folder-validation -->'; | |
| const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; | |
| const bodyParts = [ | |
| marker, | |
| 'Issues discovered by workflow `codeowners-folder-validation`.', | |
| '' | |
| ]; | |
| if (missing.length > 0) { | |
| bodyParts.push('## Missing CODEOWNERS entries', ''); | |
| bodyParts.push(...missing.map((p) => `- \`${p}\``)); | |
| bodyParts.push(''); | |
| } | |
| if (insufficientOwners.length > 0) { | |
| bodyParts.push('## Insufficient owners', ''); | |
| bodyParts.push('Each skill/test folder must have at least **2 individual owners** or **1 team**.', ''); | |
| bodyParts.push(...insufficientOwners.map((e) => `- \`${e.path}\` — current: ${e.owners}`)); | |
| bodyParts.push(''); | |
| } | |
| bodyParts.push(`Run: ${runUrl}`); | |
| const body = bodyParts.join('\n'); | |
| const { data: issues } = await github.rest.issues.listForRepo({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| state: 'open', | |
| per_page: 100 | |
| }); | |
| const existing = issues.find((i) => i.title === title); | |
| if (existing) { | |
| await github.rest.issues.update({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: existing.number, | |
| body | |
| }); | |
| core.info(`Updated issue #${existing.number}`); | |
| } else { | |
| const created = await github.rest.issues.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| title, | |
| body | |
| }); | |
| core.info(`Created issue #${created.data.number}`); | |
| } | |
| - name: Fail when CODEOWNERS validation fails | |
| if: steps.audit.outputs.has_missing == 'true' || steps.audit.outputs.has_insufficient_owners == 'true' | |
| shell: pwsh | |
| env: | |
| MISSING_JSON: ${{ steps.audit.outputs.missing_json }} | |
| INSUFFICIENT_OWNERS_JSON: ${{ steps.audit.outputs.insufficient_owners_json }} | |
| run: | | |
| if ($env:MISSING_JSON -ne '[]') { | |
| Write-Host "Missing entries: $env:MISSING_JSON" | |
| } | |
| if ($env:INSUFFICIENT_OWNERS_JSON -ne '[]') { | |
| Write-Host "Insufficient owners: $env:INSUFFICIENT_OWNERS_JSON" | |
| } | |
| Write-Error 'CODEOWNERS validation failed. Each skill/test folder needs a CODEOWNERS entry with 2+ individuals or 1+ team.' | |
| exit 1 | |