|
| 1 | +# Copyright (c) Microsoft Corporation. |
| 2 | +# Licensed under the MIT License. |
| 3 | + |
| 4 | +<# |
| 5 | + Lint rule: every KQL join must state an explicit kind (PR #2225). |
| 6 | +
|
| 7 | + A bare `| join (...)` defaults to kind=innerunique, which deduplicates the left side |
| 8 | + on the join key and silently drops rows. This has caused real data loss (savings plan |
| 9 | + recommendations collapsing to one row per subscription, SQL VMs with duplicate names |
| 10 | + disappearing). In Azure Resource Graph the same default applies and `lookup` is not |
| 11 | + available, so an explicit kind is the only way to state intent. |
| 12 | +
|
| 13 | + The rule scans every surface that carries KQL: hub database scripts, the query catalog, |
| 14 | + ARG recommendation queries, the ADX dashboard, the finops-alerts logic app, workbooks, |
| 15 | + optimization engine runbooks and views, and the published docs examples. |
| 16 | +
|
| 17 | + Known pre-existing bare joins are baselined per file below. The baseline is a ratchet: |
| 18 | + - Fixing a bare join REQUIRES lowering the count here (the test fails on stale entries). |
| 19 | + - Adding a new bare join is never allowed; write `join kind=...` explicitly. |
| 20 | +
|
| 21 | + ARG-only surfaces (workbooks, recommendation queries, the alerts logic app) are additionally |
| 22 | + checked for operators Azure Resource Graph rejects: the lookup operator and the semi/anti |
| 23 | + join flavors. Verified live against ARG (2026-08): supported kinds are inner, innerunique, |
| 24 | + leftouter, rightouter, fullouter; lookup, leftsemi, leftanti, rightsemi, rightanti, and |
| 25 | + `in`/`!in` with a subquery are all rejected with InvalidQuery. Exclusion joins in ARG must |
| 26 | + therefore use the leftouter + `where isempty(<right key>)` emulation (with a key-unique |
| 27 | + right side) — the one place that pattern is acceptable. |
| 28 | +#> |
| 29 | + |
| 30 | +Describe 'KqlJoinKinds' { |
| 31 | + |
| 32 | + BeforeDiscovery { |
| 33 | + $repoRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path |
| 34 | + |
| 35 | + $scanTargets = @( |
| 36 | + @{ Path = 'src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts'; Filter = '*.kql'; Recurse = $false } |
| 37 | + @{ Path = 'src/queries/catalog'; Filter = '*.kql'; Recurse = $false } |
| 38 | + @{ Path = 'src/powershell/Tests/assets'; Filter = '*.kql'; Recurse = $false } |
| 39 | + @{ Path = 'src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries'; Filter = '*.json'; Recurse = $false } |
| 40 | + @{ Path = 'src/templates/finops-hub'; Filter = 'dashboard.json'; Recurse = $false } |
| 41 | + @{ Path = 'src/templates/finops-alerts/modules'; Filter = 'logicApp.bicep'; Recurse = $false } |
| 42 | + @{ Path = 'src/workbooks'; Filter = '*.workbook'; Recurse = $true } |
| 43 | + @{ Path = 'src/workbooks'; Filter = 'workbook.json'; Recurse = $true } |
| 44 | + @{ Path = 'src/optimization-engine/runbooks'; Filter = '*.ps1'; Recurse = $true } |
| 45 | + @{ Path = 'src/optimization-engine/views'; Filter = '*.json'; Recurse = $true } |
| 46 | + @{ Path = 'docs-mslearn'; Filter = '*.md'; Recurse = $true } |
| 47 | + ) |
| 48 | + |
| 49 | + $scanFiles = @($scanTargets | ForEach-Object { |
| 50 | + $full = Join-Path $repoRoot $_.Path |
| 51 | + Get-ChildItem -Path $full -Filter $_.Filter -Recurse:$_.Recurse -File -ErrorAction SilentlyContinue |
| 52 | + } | Sort-Object FullName -Unique | ForEach-Object { |
| 53 | + @{ Name = $_.Name; FullName = $_.FullName; RelPath = $_.FullName.Substring($repoRoot.Length + 1).Replace('\', '/') } |
| 54 | + }) |
| 55 | + |
| 56 | + # Surfaces whose KQL runs on Azure Resource Graph. Workbook files may also contain the |
| 57 | + # occasional Log Analytics query (queryType 0); if one legitimately needs lookup or a |
| 58 | + # semi/anti join, add a per-file allowlist analogous to the bare-join baseline. |
| 59 | + $argFiles = @($scanFiles | Where-Object { |
| 60 | + $_.RelPath -like 'src/workbooks/*' -or |
| 61 | + $_.RelPath -like 'src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/*' -or |
| 62 | + $_.RelPath -eq 'src/templates/finops-alerts/modules/logicApp.bicep' |
| 63 | + }) |
| 64 | + |
| 65 | + # Published docs mix engines within a single file: docs-mslearn/best-practices/compute.md |
| 66 | + # carries both ARG inventory queries and hub (ADX) cost queries, and the latter legitimately |
| 67 | + # use lookup. So docs are classified per code block rather than per file - see the |
| 68 | + # 'ARG examples' test below. |
| 69 | + $docsFiles = @($scanFiles | Where-Object { $_.RelPath -like 'docs-mslearn/*' }) |
| 70 | + } |
| 71 | + |
| 72 | + BeforeAll { |
| 73 | + $repoRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path |
| 74 | + $scanFileCount = @( |
| 75 | + (Join-Path $repoRoot 'src/workbooks'), |
| 76 | + (Join-Path $repoRoot 'src/optimization-engine'), |
| 77 | + (Join-Path $repoRoot 'src/queries/catalog'), |
| 78 | + (Join-Path $repoRoot 'src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts') |
| 79 | + ) | ForEach-Object { Get-ChildItem -Path $_ -Recurse -Include '*.kql', '*.workbook', 'workbook.json', '*.ps1', '*.json' -File -ErrorAction SilentlyContinue } | Measure-Object | Select-Object -ExpandProperty Count |
| 80 | + |
| 81 | + # Matches `| join` not followed by `kind=` before the right-table parenthesis. |
| 82 | + # Catches `| join (`, `| join(`, and `| join hint.x=y (`; ignores `| join kind=...` and `lookup`. |
| 83 | + $bareJoinPattern = [regex]'\|\s*join\b(?![^(\r\n]*\bkind\s*=)' |
| 84 | + |
| 85 | + # Operators Azure Resource Graph rejects with InvalidQuery (verified live, 2026-08). |
| 86 | + $argRejectedPattern = [regex]'\|\s*lookup\b|join\s+kind\s*=\s*(leftanti|leftsemi|rightanti|rightsemi|anti|semi|leftantisemi|rightantisemi)\b' |
| 87 | + |
| 88 | + # ARG tables that can open a query. A KQL query names its source table first, so the first |
| 89 | + # non-comment line of a docs code block identifies the engine it targets. |
| 90 | + $argTablePattern = [regex]'^\s*(resources|resourcecontainers|advisorresources|resourcechanges|resourcecontainerchanges|healthresources|securityresources|policyresources|guestconfigurationresources|patchassessmentresources|patchinstallationresources|maintenanceresources|servicehealthresources|desktopvirtualizationresources|kubernetesconfigurationresources|extendedlocationresources|networkresources|chaosresources|iotsecurityresources|insightsresources)\b' |
| 91 | + |
| 92 | + # Pre-existing bare joins, counted per repo-relative path. Ratchet only: lower on fix, never raise. |
| 93 | + # All remaining entries are benign today (left side unique on the join key) but rely on the |
| 94 | + # innerunique default implicitly. Convert to an explicit kind when touching these queries. |
| 95 | + $baseline = @{ |
| 96 | + 'src/workbooks/optimization/AHB/AHB.workbook' = 24 |
| 97 | + 'src/workbooks/optimization/Compute/AHB.workbook' = 20 |
| 98 | + 'src/workbooks/optimization/Networking/Networking.workbook' = 3 |
| 99 | + 'src/workbooks/governance/workbook.json' = 1 |
| 100 | + } |
| 101 | + } |
| 102 | + |
| 103 | + It 'Should scan at least one file per surface' { |
| 104 | + $scanFileCount | Should -BeGreaterThan 100 |
| 105 | + } |
| 106 | + |
| 107 | + It 'Should not add bare joins (no explicit kind): <RelPath>' -ForEach $scanFiles { |
| 108 | + $content = Get-Content -Path $FullName -Raw |
| 109 | + $bareJoins = @($bareJoinPattern.Matches($content)) |
| 110 | + $allowed = if ($baseline.ContainsKey($RelPath)) { $baseline[$RelPath] } else { 0 } |
| 111 | + |
| 112 | + $bareJoins.Count | Should -BeLessOrEqual $allowed -Because ('a bare "| join" defaults to kind=innerunique, which deduplicates the left side on the join key and silently drops rows (see PR #2225). State the kind explicitly: kind=inner for lookups/filters, kind=leftouter for enrichment, kind=leftanti for exclusion. In ADX/Log Analytics, prefer the lookup operator for small dimension tables.') |
| 113 | + |
| 114 | + if ($bareJoins.Count -le $allowed) |
| 115 | + { |
| 116 | + # Ratchet: if bare joins were removed, the baseline must be lowered so they cannot return. |
| 117 | + $bareJoins.Count | Should -Be $allowed -Because ("the bare-join count in this file dropped below the baseline ($allowed); lower the baseline entry for '$RelPath' in KqlJoinKinds.Tests.ps1 to $($bareJoins.Count) (or remove it if 0) so the fix is locked in.") |
| 118 | + } |
| 119 | + } |
| 120 | + |
| 121 | + It 'Should not use operators ARG rejects (lookup, semi/anti joins): <RelPath>' -ForEach $argFiles { |
| 122 | + $content = Get-Content -Path $FullName -Raw |
| 123 | + $rejected = @($argRejectedPattern.Matches($content)) |
| 124 | + |
| 125 | + @($rejected | ForEach-Object { $_.Value }) -join '; ' | Should -BeNullOrEmpty -Because ('Azure Resource Graph rejects the lookup operator and all semi/anti join flavors with InvalidQuery (verified live; supported kinds are inner, innerunique, leftouter, rightouter, fullouter). For exclusions in ARG, use join kind=leftouter + where isempty(<right key>) with a key-unique right side. If this file contains a Log Analytics query that legitimately needs the operator, add a per-file allowlist to this test.') |
| 126 | + } |
| 127 | + |
| 128 | + It 'Should not use operators ARG rejects in docs ARG examples: <RelPath>' -ForEach $docsFiles { |
| 129 | + $content = Get-Content -Path $FullName -Raw |
| 130 | + |
| 131 | + # Fenced code blocks, so a hub (ADX) example in the same file cannot mask or trip this rule. |
| 132 | + $offenders = @( |
| 133 | + foreach ($block in [regex]::Matches($content, '(?ms)^```[a-zA-Z]*\r?\n(.*?)^```')) |
| 134 | + { |
| 135 | + $code = $block.Groups[1].Value |
| 136 | + $firstLine = @($code -split '\r?\n' | Where-Object { $_.Trim() -and $_.Trim() -notmatch '^//' })[0] |
| 137 | + if ($null -eq $firstLine -or -not $argTablePattern.IsMatch($firstLine)) { continue } |
| 138 | + |
| 139 | + $argRejectedPattern.Matches($code) | ForEach-Object { $_.Value.Trim() } |
| 140 | + } |
| 141 | + ) |
| 142 | + |
| 143 | + $offenders -join '; ' | Should -BeNullOrEmpty -Because ('this code block opens with an Azure Resource Graph table, and ARG rejects the lookup operator and all semi/anti join flavors with InvalidQuery (verified live). Published examples are copied verbatim by readers, so they must run as written: use join kind=leftouter + where isempty(<right key>) for exclusions. Hub (ADX) examples in the same file are unaffected - they open with Costs, Prices, or another hub table.') |
| 144 | + } |
| 145 | +} |
![github-actions[bot]](https://avatars.githubusercontent.com/in/15368?v=4&size=40){kind=avatar}
0 commit comments