diff --git a/.gitignore b/.gitignore
index 189b956d5..835f03ec4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -368,3 +368,8 @@ venv/
ENV/
env/
+# Claude settings
+.claude/
+claude.settings.local.json
+settings.local.json
+
diff --git a/.vscode/settings.json b/.vscode/settings.json
index 574991ca3..6b6db9a8d 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -1,4 +1,23 @@
{
+ "[bicep]": {
+ "editor.formatOnSave": true,
+ "editor.defaultFormatter": "esbenp.prettier-vscode"
+ },
+ "[json]": {
+ "editor.formatOnSave": true,
+ "editor.defaultFormatter": "esbenp.prettier-vscode"
+ },
+ "[markdown]": {
+ "editor.formatOnSave": true,
+ "editor.defaultFormatter": "esbenp.prettier-vscode"
+ },
+ "[powerquery]": {
+ "editor.formatOnSave": false
+ },
+ "[powershell]": {
+ "editor.formatOnSave": true,
+ "editor.defaultFormatter": "ms-vscode.powershell"
+ },
"cSpell.words": [
"ADLS",
"architecting",
@@ -51,10 +70,15 @@
"Unhide",
"westus"
],
- "markdown.preview.typographer": false,
+ "editor.detectIndentation": false,
+ "editor.formatOnSave": true,
+ "editor.codeActionsOnSave": {
+ "source.fixAll": "explicit"
+ },
"markdown.extension.italic.indicator": "_",
"markdown.extension.toc.levels": "2..2",
"markdown.extension.toc.orderedList": false,
+ "markdown.preview.typographer": false,
"markdown.updateLinksOnFileMove.enabled": "prompt",
"markdownlint.config": {
"line-length": false,
@@ -70,9 +94,13 @@
"powershell.codeFormatting.whitespaceBeforeOpenBrace": true,
"powershell.codeFormatting.whitespaceBeforeOpenParen": true,
"powershell.codeFormatting.whitespaceInsideBrace": true,
- "[powerquery]": {
- "editor.formatOnSave": false
- },
"powerquery.general.experimental": false,
- "editor.detectIndentation": false
+ "terminal.integrated.defaultProfile.osx": "pwsh",
+ "terminal.integrated.profiles.osx": {
+ "pwsh": {
+ "path": "pwsh",
+ "args": ["-NoLogo"]
+ }
+ },
+ "workbench.editor.enablePreview": false
}
diff --git a/CLAUDE.md b/CLAUDE.md
new file mode 100644
index 000000000..3de4262b1
--- /dev/null
+++ b/CLAUDE.md
@@ -0,0 +1,182 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Repository Overview
+
+The FinOps Toolkit is an open-source collection of tools for adopting and implementing FinOps capabilities in the Microsoft Cloud. It contains templates, PowerShell modules, workbooks, optimization engines, and supporting documentation organized in a modular architecture.
+
+## Common Commands
+
+### Building and Development
+
+```bash
+# Build entire toolkit
+npm run build
+# or
+pwsh -Command ./src/scripts/Build-Toolkit
+
+# Build FinOps hubs
+pwsh -Command ./src/scripts/Build-Toolkit finops-hub
+
+# Build specific components
+npm run build-ps # PowerShell module only
+pwsh -Command ./src/scripts/Build-Bicep # Bicep templates
+pwsh -Command ./src/scripts/Build-Workbook # Azure Monitor workbooks
+pwsh -Command ./src/scripts/Build-OpenData # Open data files
+
+# Deploy for testing
+npm run deploy-test
+# or
+pwsh -Command ./src/scripts/Deploy-Toolkit -Build -Test
+
+# Package for release
+npm run package
+# or
+pwsh -Command ./src/scripts/Package-Toolkit -Build
+```
+
+### Testing
+
+```bash
+# Run PowerShell unit tests
+npm run pester
+# or
+pwsh -Command Invoke-Pester -Output Detailed -Path ./src/powershell/Tests/Unit/*
+
+# Run integration tests
+pwsh -Command ./src/scripts/Test-PowerShell -Integration
+
+# Run specific test categories
+pwsh -Command ./src/scripts/Test-PowerShell -Hubs -Exports
+
+# Lint PowerShell code
+pwsh -Command ./src/scripts/Test-PowerShell -Lint
+```
+
+### Bicep Development
+
+```bash
+# Validate Bicep templates
+bicep build path/to/template.bicep --stdout
+
+# Test template deployment
+az deployment group what-if --resource-group myRG --template-file template.bicep
+```
+
+## Architecture and Code Organization
+
+### High-Level Structure
+
+- **`/src/templates/`** - ARM/Bicep infrastructure templates with modular namespace organization
+- **`/src/powershell/`** - PowerShell module with public/private functions and comprehensive tests
+- **`/src/optimization-engine/`** - Azure Optimization Engine for cost recommendations
+- **`/src/workbooks/`** - Azure Monitor workbooks for governance and optimization
+- **`/src/open-data/`** - Reference data (pricing, regions, services) with utilities
+- **`/src/scripts/`** - Build automation and development tools
+- **`/docs/`** - Jekyll documentation website
+- **`/docs-mslearn/`** - Microsoft Learn documentation website
+- **`/docs-wiki/`** - GitHub wiki documentation
+
+### Current Architectural Reorganization
+
+The FinOps hubs solution is actively migrating to a namespace-based modular structure:
+
+- **`Microsoft.FinOpsHubs/`** - Core FinOps Hub infrastructure modules
+- **`Microsoft.CostManagement/`** - Cost management exports and schemas
+- **`fx/`** - Shared foundation components (hub-types, scripts, utilities)
+
+### Template Architecture
+
+Templates use a multi-target build system that generates:
+
+- Azure Quickstart Templates (ARM JSON)
+- Bicep Registry modules
+- Standalone deployments
+- Azure portal UI definitions
+
+Key patterns:
+
+- **`.build.config`** files control build behavior per template
+- **`settings.json`** contains component-specific configuration
+- **`ftkver.txt`** files maintain version synchronization
+- **Conditional resource deployment** based on parameters
+
+### PowerShell Module Structure
+
+- **`Public/`** - User-facing cmdlets (Get-_, Set-_, New-\*, etc.)
+- **`Private/`** - Internal utilities and helpers
+- **`Tests/Unit/`** - Pester unit tests with mocking
+- **`Tests/Integration/`** - End-to-end Azure integration tests
+- **Module manifest** defines exports and dependencies
+
+### Data Flow and Integration
+
+- **Open data** provides reference information consumed by templates and PowerShell
+- **Build scripts** orchestrate compilation across all components
+- **Version management** is centralized through `Update-Version.ps1`
+- **Templates reference** shared schemas and types from `fx/` namespace
+
+## Key Development Patterns
+
+### Template Development
+
+- Use `newApp()` and `newHub()` functions from `fx/hub-types.bicep` for consistent resource naming
+- Follow the conditional deployment pattern: `resource foo 'type' = if (condition) { ... }`
+- Implement proper parameter validation with `@allowed`, `@minValue`, `@maxValue`
+- Include telemetry tracking via `defaultTelemetry` parameter
+
+### PowerShell Development
+
+- All public functions must have comment-based help
+- Use approved verbs from `Get-Verb`
+- Implement comprehensive parameter validation
+- Support `-WhatIf` and `-Confirm` for destructive operations
+- Include Pester tests for all functions
+
+### Testing Strategy
+
+- **Lint tests** validate syntax and coding standards
+- **Unit tests** test isolated function behavior with mocks
+- **Integration tests** perform end-to-end validation against Azure
+- **Template validation** uses `bicep build` and ARM what-if deployments
+
+### Build System Integration
+
+The PowerShell-based build system:
+
+- Compiles templates to multiple target formats
+- Validates all code before packaging
+- Maintains version consistency across components
+- Generates release artifacts automatically
+
+### Version Management
+
+- Central version in `package.json` (currently 12.0.0)
+- Synchronized across all components via build scripts
+- Individual `ftkver.txt` files distributed to modules
+- Git tags correspond to release versions
+
+## Repository Conventions
+
+### Branch Strategy
+
+- **`dev`** - Main integration branch
+- Feature branches merge into `dev`
+- Releases are tagged from `dev`
+
+### File Organization
+
+- Templates follow namespace/module/component structure
+- PowerShell follows standard module layout
+- Documentation uses Jekyll conventions
+- Build artifacts are generated, not checked in
+
+### Coding Standards
+
+- Always follow the content and coding standards defined in `docs-wiki/Coding-guidelines.md`
+- Content (text strings): Follow the Microsoft style guide and always use sentence casing except for proper nouns
+- Bicep: Follow Azure Bicep style guide
+- PowerShell: Use PowerShell best practices and approved verbs
+- Documentation: Use markdown with consistent formatting
+- Commit messages: Use conventional commit format
diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md
index 4e4134883..c9b805198 100644
--- a/docs-mslearn/toolkit/changelog.md
+++ b/docs-mslearn/toolkit/changelog.md
@@ -13,7 +13,8 @@ ms.reviewer: micflan
-
+
+
# FinOps toolkit changelog
This article summarizes the features and enhancements in each release of the FinOps toolkit.
@@ -33,32 +34,29 @@ The following section lists features and enhancements that are currently in deve
- Cost Management export modules for subscriptions and resource groups.
-### Documentation improvements
-
+
-### [Power BI reports](power-bi/reports.md) v13
+## v13
-- **Fixed**
- - Fixed tag expansion in Power BI reports when tag names contain special characters like colons.
+_Released August 2025_
### [FinOps hubs](hubs/finops-hubs-overview.md) v13
- **Changed**
+ - Reorganized Bicep modules into separate apps.
- Enhanced [Configure scopes documentation](hubs/configure-scopes.md) to explicitly clarify that FinOps hubs support:
- Multiple Azure scopes (billing accounts, subscriptions, resource groups) in a single hub instance
- Cross-cloud data ingestion through FOCUS format support
+- **Fixed**
+ - Fixed all Bicep compilation errors and warnings with inline suppressions and descriptive comments.
+ - Fixed Build-Toolkit.ps1 bicep generate-params command bug.
-
-
-## v13
-
-_Released August 2025_
+### [Power BI reports](power-bi/reports.md) v13
+- **Fixed**
+ - Fixed tag expansion in Power BI reports when tag names contain special characters like colons.
-> [!div class="nextstepaction"]
-> [Download](https://github.com/microsoft/finops-toolkit/releases/tag/v13)
-> [!div class="nextstepaction"]
-> [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v12...v13)
+> [!div class="nextstepaction"] > [Download](https://github.com/microsoft/finops-toolkit/releases/tag/v13) > [!div class="nextstepaction"] > [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v12...v13)
@@ -162,10 +160,7 @@ _Released July 16, 2025_
- microsoft.durabletask/schedulers
- microsoft.edge/contexts
-> [!div class="nextstepaction"]
-> [Download](https://github.com/microsoft/finops-toolkit/releases/tag/v12)
-> [!div class="nextstepaction"]
-> [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v0.11...v12)
+> [!div class="nextstepaction"] > [Download](https://github.com/microsoft/finops-toolkit/releases/tag/v12) > [!div class="nextstepaction"] > [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v0.11...v12)
@@ -298,11 +293,8 @@ _Released June 2, 2025_
- microsoft.synapse/workspaces/kustopools
- microsoft.synapse/workspaces/sqlpools
- microsoft.web/sites/slots
-
-> [!div class="nextstepaction"]
-> [Download](https://github.com/microsoft/finops-toolkit/releases/tag/v0.11)
-> [!div class="nextstepaction"]
-> [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v0.10...v0.11)
+
+> [!div class="nextstepaction"] > [Download](https://github.com/microsoft/finops-toolkit/releases/tag/v0.11) > [!div class="nextstepaction"] > [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v0.10...v0.11)
@@ -406,7 +398,7 @@ _Released May 4, 2025_
- microsoft.sentinelplatformservices/sentinelplatformservices
- oracle.database/networkanchors
- oracle.database/resourceanchors
- **Changed**
+- **Changed**
- Updated the following resource types:
- dell.storage/filesystems
- lambdatest.hyperexecute/organizations
@@ -419,10 +411,7 @@ _Released May 4, 2025_
- microsoft.liftrpilot/organizations
- mongodb.atlas/organizations
-> [!div class="nextstepaction"]
-> [Download](https://github.com/microsoft/finops-toolkit/releases/tag/v0.10)
-> [!div class="nextstepaction"]
-> [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v0.9...v0.10)
+> [!div class="nextstepaction"] > [Download](https://github.com/microsoft/finops-toolkit/releases/tag/v0.10) > [!div class="nextstepaction"] > [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v0.9...v0.10)
@@ -436,7 +425,7 @@ This release is a minor patch to fix the FinOps hub deployment and the Power BI
- **Fixed**
- Removed a reference to old columns that are no longer applicable.
- - This may have caused new deployments to fail on April 4. Upgrades were not affected.
+ - This may have caused new deployments to fail on April 4. Upgrades were not affected.
### [Power BI reports](power-bi/reports.md) v0.9 Update 1
@@ -605,10 +594,7 @@ _Released April 4, 2025_
- **Added**
- Added sample data for MCA reservation exports.
-> [!div class="nextstepaction"]
-> [Download](https://github.com/microsoft/finops-toolkit/releases/tag/v0.9)
-> [!div class="nextstepaction"]
-> [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v0.8...v0.9)
+> [!div class="nextstepaction"] > [Download](https://github.com/microsoft/finops-toolkit/releases/tag/v0.9) > [!div class="nextstepaction"] > [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v0.8...v0.9)
@@ -667,6 +653,7 @@ _Released February 12, 2025_
### [FinOps hubs](hubs/finops-hubs-overview.md) v0.8
+
- **Added**
- Added Data Explorer dashboard template.
- Added new KQL functions in Data Explorer:
@@ -700,10 +687,10 @@ _Released February 12, 2025_
#### [Optimization workbook](workbooks/optimization.md) v0.8
- **Added**
- - Azure Arc Windows license management under the **Commitment Discounts** tab.
+ - Azure Arc Windows license management under the **Commitment Discounts** tab.
- **Fixed**
- Enabled "Export to CSV" option on the **Idle backups** query.
- - Corrected VM processor details on the **Compute** tab query.
+ - Corrected VM processor details on the **Compute** tab query.
### [Optimization engine](optimization-engine/overview.md) v0.8
@@ -819,10 +806,7 @@ _Released February 12, 2025_
- microsoft.iotoperations/instances
- microsoft.networkcloud/baremetalmachines
-> [!div class="nextstepaction"]
-> [Download](https://github.com/microsoft/finops-toolkit/releases/tag/v0.8)
-> [!div class="nextstepaction"]
-> [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v0.7...v0.8)
+> [!div class="nextstepaction"] > [Download](https://github.com/microsoft/finops-toolkit/releases/tag/v0.8) > [!div class="nextstepaction"] > [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v0.7...v0.8)
@@ -875,7 +859,7 @@ _Released December 1, 2024_
- Updated supported spend estimates in the Power BI documentation.
- **Fixed**
- Fixed EffectiveCost for savings plan purchases to work around a bug in exported data.
-
+
### [FinOps hubs](hubs/finops-hubs-overview.md) v0.7
_**Breaking change**_
@@ -977,10 +961,7 @@ _**Breaking change**_
- microsoft.healthdataaiservices/deidservices
- microsoft.insights/datacollectionrules
-> [!div class="nextstepaction"]
-> [Download](https://github.com/microsoft/finops-toolkit/releases/tag/v0.7)
-> [!div class="nextstepaction"]
-> [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v0.6...v0.7)
+> [!div class="nextstepaction"] > [Download](https://github.com/microsoft/finops-toolkit/releases/tag/v0.7) > [!div class="nextstepaction"] > [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v0.6...v0.7)
@@ -1130,10 +1111,7 @@ _Released October 2, 2024_
- microsoft.sql/longtermretentionservers
- microsoft.verifiedid/authorities
-> [!div class="nextstepaction"]
-> [Download v0.6](https://github.com/microsoft/finops-toolkit/releases/tag/v0.6)
-> [!div class="nextstepaction"]
-> [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v0.5...v0.6)
+> [!div class="nextstepaction"] > [Download v0.6](https://github.com/microsoft/finops-toolkit/releases/tag/v0.6) > [!div class="nextstepaction"] > [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v0.5...v0.6)
@@ -1365,7 +1343,7 @@ _Released September 1, 2024_
- Updated multiple resource types for the following resource providers: **microsoft.awsconnector**.
- Changed the following resource providers to be GA: **microsoft.modsimworkbench**.
- **Removed**
- - Removed internal "microsoft.cognitiveservices/browse*" resource types.
+ - Removed internal "microsoft.cognitiveservices/browse\*" resource types.
#### [Services v0.5](open-data.md#services)
@@ -1402,10 +1380,7 @@ _Released September 1, 2024_
- Move Microsoft Defender for Endpoint from the **Multicloud** service category to **Security**.
- Move StorSimple from the **Multicloud** service category to **Storage**.
-> [!div class="nextstepaction"]
-> [Download v0.5](https://github.com/microsoft/finops-toolkit/releases/tag/v0.5)
-> [!div class="nextstepaction"]
-> [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v0.4...v0.5)
+> [!div class="nextstepaction"] > [Download v0.5](https://github.com/microsoft/finops-toolkit/releases/tag/v0.5) > [!div class="nextstepaction"] > [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v0.4...v0.5)
@@ -1527,10 +1502,7 @@ _Released July 12, 2024_
- Changed the primary columns in the [Regions](open-data.md#regions) and [Services](open-data.md#services) open data files to be lowercase.
- Updated all [sample exports](open-data.md#dataset-examples) to use the same date range as the FOCUS 1.0 dataset.
-> [!div class="nextstepaction"]
-> [Download v0.4](https://github.com/microsoft/finops-toolkit/releases/tag/v0.4)
-> [!div class="nextstepaction"]
-> [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v0.3...v0.4)
+> [!div class="nextstepaction"] > [Download v0.4](https://github.com/microsoft/finops-toolkit/releases/tag/v0.4) > [!div class="nextstepaction"] > [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v0.3...v0.4)
@@ -1618,10 +1590,7 @@ _Released March 28, 2024_
- Added ServiceModel and Environment columns to the [services](open-data.md#services) data ([#585](https://github.com/microsoft/finops-toolkit/issues/585)).
- New and updated [resource types](open-data.md#resource-types) and icons.
-> [!div class="nextstepaction"]
-> [Download v0.3](https://github.com/microsoft/finops-toolkit/releases/tag/v0.3)
-> [!div class="nextstepaction"]
-> [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v0.2...v0.3)
+> [!div class="nextstepaction"] > [Download v0.3](https://github.com/microsoft/finops-toolkit/releases/tag/v0.3) > [!div class="nextstepaction"] > [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v0.2...v0.3)
@@ -1710,10 +1679,7 @@ _**Breaking change**_
- **Added**
- [FinOps Open Cost and Usage Specification (FOCUS) details](../focus/what-is-focus.md).
-> [!div class="nextstepaction"]
-> [Download v0.2](https://github.com/microsoft/finops-toolkit/releases/tag/v0.2)
-> [!div class="nextstepaction"]
-> [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v0.1.1...v0.2)
+> [!div class="nextstepaction"] > [Download v0.2](https://github.com/microsoft/finops-toolkit/releases/tag/v0.2) > [!div class="nextstepaction"] > [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v0.1.1...v0.2)
@@ -1748,10 +1714,7 @@ _Released October 26, 2023_
- [Register-FinOpsHubProviders](powershell/hubs/Register-FinOpsHubProviders.md)
- [Remove-FinOpsHub](powershell/hubs/Remove-FinOpsHub.md)
-> [!div class="nextstepaction"]
-> [Download v0.1.1](https://github.com/microsoft/finops-toolkit/releases/tag/v0.1.1)
-> [!div class="nextstepaction"]
-> [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v0.1...v0.1.1)
+> [!div class="nextstepaction"] > [Download v0.1.1](https://github.com/microsoft/finops-toolkit/releases/tag/v0.1.1) > [!div class="nextstepaction"] > [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v0.1...v0.1.1)
@@ -1799,10 +1762,7 @@ _Released October 22, 2023_
- [Regions](open-data.md#regions) to map historical resource location values in Microsoft Cost Management to standard Azure regions.
- [Services](open-data.md#services) to map all resource types to FOCUS service names and categories.
-> [!div class="nextstepaction"]
-> [Download v0.1](https://github.com/microsoft/finops-toolkit/releases/tag/v0.1)
-> [!div class="nextstepaction"]
-> [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v0.0.1...v0.1)
+> [!div class="nextstepaction"] > [Download v0.1](https://github.com/microsoft/finops-toolkit/releases/tag/v0.1) > [!div class="nextstepaction"] > [Full changelog](https://github.com/microsoft/finops-toolkit/compare/v0.0.1...v0.1)
@@ -1827,10 +1787,7 @@ _Released May 27, 2023_
- **Added**
- [Cost optimization workbook](workbooks/optimization.md) to centralize cost optimization.
-> [!div class="nextstepaction"]
-> [Download v0.0.1](https://github.com/microsoft/finops-toolkit/releases/tag/v0.0.1)
-> [!div class="nextstepaction"]
-> [Full changelog](https://github.com/microsoft/finops-toolkit/compare/878e4864ca785db4fc13bdd2ec3a6a00058688c3...v0.0.1)
+> [!div class="nextstepaction"] > [Download v0.0.1](https://github.com/microsoft/finops-toolkit/releases/tag/v0.0.1) > [!div class="nextstepaction"] > [Full changelog](https://github.com/microsoft/finops-toolkit/compare/878e4864ca785db4fc13bdd2ec3a6a00058688c3...v0.0.1)
@@ -1838,12 +1795,10 @@ _Released May 27, 2023_
Let us know how we're doing with a quick review. We use these reviews to improve and expand FinOps tools and resources.
-> [!div class="nextstepaction"]
-> [Give feedback](https://portal.azure.com/#view/HubsExtension/InProductFeedbackBlade/extensionName/FinOpsToolkit/cesQuestion/How%20easy%20or%20hard%20is%20it%20to%20use%20FinOps%20toolkit%20tools%20and%20resources%3F/cvaQuestion/How%20valuable%20is%20the%20FinOps%20toolkit%3F/surveyId/FTK/bladeName/Toolkit/featureName/Changelog)
+> [!div class="nextstepaction"] > [Give feedback](https://portal.azure.com/#view/HubsExtension/InProductFeedbackBlade/extensionName/FinOpsToolkit/cesQuestion/How%20easy%20or%20hard%20is%20it%20to%20use%20FinOps%20toolkit%20tools%20and%20resources%3F/cvaQuestion/How%20valuable%20is%20the%20FinOps%20toolkit%3F/surveyId/FTK/bladeName/Toolkit/featureName/Changelog)
If you're looking for something specific, vote for an existing or create a new idea. Share ideas with others to get more votes. We focus on ideas with the most votes.
-> [!div class="nextstepaction"]
-> [Vote on or suggest ideas](https://github.com/microsoft/finops-toolkit/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%252B1-desc)
+> [!div class="nextstepaction"] > [Vote on or suggest ideas](https://github.com/microsoft/finops-toolkit/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%252B1-desc)
diff --git a/src/scripts/Build-OpenData.ps1 b/src/scripts/Build-OpenData.ps1
index be8ffe2ec..c0a961a45 100644
--- a/src/scripts/Build-OpenData.ps1
+++ b/src/scripts/Build-OpenData.ps1
@@ -173,7 +173,7 @@ function Write-KqlWrapperFunction($Function, $Parts)
Write-Output "}"
}
-$hubsDir = "$PSScriptRoot/../templates/finops-hub/modules/scripts"
+$hubsDir = "$PSScriptRoot/../templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts"
$psDir = "$PSScriptRoot/../powershell"
$srcDir = "$PSScriptRoot/../open-data"
$svgDir = "$PSScriptRoot/../../docs/svg"
diff --git a/src/scripts/Build-Toolkit.ps1 b/src/scripts/Build-Toolkit.ps1
index fbee8dd7e..7cad6cf61 100644
--- a/src/scripts/Build-Toolkit.ps1
+++ b/src/scripts/Build-Toolkit.ps1
@@ -21,6 +21,8 @@
.LINK
https://github.com/microsoft/finops-toolkit/blob/dev/src/scripts/README.md#-build-toolkit
#>
+
+[CmdletBinding()]
param(
[Parameter(Position = 0)][string]$Template = "*",
[switch]$Major,
@@ -32,11 +34,15 @@ param(
# Create output directory
$outDir = "$PSScriptRoot/../../release"
-& "$PSScriptRoot/New-Directory" $outDir
+Write-Verbose "Creating output directory: $outDir"
+& "$PSScriptRoot/New-Directory.ps1" $outDir
+
+Write-Verbose "Starting build for template pattern: '$Template'"
# Update version
Write-Host ''
-$ver = & "$PSScriptRoot/Update-Version" -Major:$Major -Minor:$Minor -Patch:$Patch -Prerelease:$Prerelease -Label $Label
+Write-Verbose "Updating version information..."
+$ver = & "$PSScriptRoot/Update-Version.ps1" -Major:$Major -Minor:$Minor -Patch:$Patch -Prerelease:$Prerelease -Label $Label
if ($Major -or $Minor -or $Patch -or $Prerelease)
{
Write-Host "Updated version to $ver"
@@ -48,56 +54,109 @@ else
Write-Host ''
# Generate Bicep Registry modules
-Get-ChildItem "$PSScriptRoot/../bicep-registry/$($Template -replace '(subscription|resourceGroup|managementGroup|tenant)-', '')*" -Directory -ErrorAction SilentlyContinue `
+Write-Verbose "Searching for Bicep Registry modules..."
+$bicepModules = Get-ChildItem "$PSScriptRoot/../bicep-registry/$($Template -replace '(subscription|resourceGroup|managementGroup|tenant)-', '')*" -Directory -ErrorAction SilentlyContinue `
| Where-Object { $_.Name -ne '.scaffold' }
-| ForEach-Object {
- ./Build-Bicep $_.Name
+
+if ($bicepModules)
+{
+ Write-Verbose "Found $($bicepModules.Count) Bicep Registry module(s) to build"
+ $bicepModules | ForEach-Object {
+ Write-Verbose "Building Bicep module: $($_.Name)"
+ & "$PSScriptRoot/Build-Bicep.ps1" $_.Name
+ }
+}
+else
+{
+ Write-Verbose "No Bicep Registry modules found matching pattern: $Template"
}
# Generate deployment files from main.bicep in the target directory
function Build-MainBicep($dir)
{
Write-Host " Generating parameters..."
+ Write-Verbose " Building Bicep template: $dir/main.bicep"
bicep build "$dir/main.bicep" --outfile "$dir/azuredeploy.json"
- bicep generate-params "$dir/main.bicep" --outfile "$dir/azuredeploy.json"
+
+ Write-Verbose " Generating parameter template: $dir/azuredeploy.parameters.json"
+ bicep generate-params "$dir/main.bicep" --outfile "$dir/azuredeploy.parameters.json"
+
$paramFilePath = "$dir/azuredeploy.parameters.json"
+ Write-Verbose " Processing parameter placeholders in: $paramFilePath"
$params = Get-Content $paramFilePath -Raw | ConvertFrom-Json
- $params.parameters.psobject.Properties `
- | ForEach-Object {
+
+ $parameterCount = 0
+ $params.parameters.psobject.Properties | ForEach-Object {
# Add placeholder values for required parameters
# See AQT docs for allowed values: https://github.com/Azure/azure-quickstart-templates/tree/4a6e5eae3c860208bf1731b392ae2b8a5fb24f4b/1-CONTRIBUTION-GUIDE#azure-devops-ci
- if ($_ -and $_.Name.EndsWith('Name')) { $_.Value.value = "GEN-UNIQUE" }
+ if ($_ -and $_.Name.EndsWith('Name'))
+ {
+ Write-Verbose " Setting placeholder for parameter: $($_.Name)"
+ $_.Value.value = "GEN-UNIQUE"
+ $parameterCount++
+ }
}
+
+ Write-Verbose " Updated $parameterCount parameter placeholder(s)"
$params | ConvertTo-Json -Depth 100 | Out-File $paramFilePath
}
# Generate workbook templates
-Get-ChildItem "$PSScriptRoot/../workbooks/*" -Directory `
+Write-Verbose "Searching for workbook templates..."
+$workbooks = Get-ChildItem "$PSScriptRoot/../workbooks/*" -Directory `
| Where-Object { $_.Name -ne '.scaffold' -and ($Template -eq "*" -or $Template -eq $_.Name -or $Template -eq "$($_.Name)-workbook" -or $Template -eq "finops-workbooks") }
-| ForEach-Object {
- $workbook = $_.Name
- Write-Host "Building workbook $workbook..."
- & "$PSScriptRoot/Build-Workbook" $workbook
- Build-MainBicep "$outdir/$workbook-workbook"
- $ver | Out-File "$outdir/$workbook-workbook/ftkver.txt" -NoNewline
- Write-Host ''
+
+if ($workbooks)
+{
+ Write-Verbose "Found $($workbooks.Count) workbook template(s) to build"
+ $workbooks | ForEach-Object {
+ $workbook = $_.Name
+ Write-Host "Building workbook $workbook..."
+ Write-Verbose " Building workbook: $workbook"
+ & "$PSScriptRoot/Build-Workbook.ps1" $workbook
+
+ Write-Verbose " Generating deployment files for: $workbook-workbook"
+ Build-MainBicep "$outdir/$workbook-workbook"
+
+ Write-Verbose " Writing version file: $outdir/$workbook-workbook/ftkver.txt"
+ $ver | Out-File "$outdir/$workbook-workbook/ftkver.txt" -NoNewline
+ Write-Host ''
+ }
+}
+else
+{
+ Write-Verbose "No workbook templates found matching pattern: $Template"
}
# Package templates
-Get-ChildItem -Path "$PSScriptRoot/../templates/*", "$PSScriptRoot/../optimization-engine*" -Directory -ErrorAction SilentlyContinue `
-| ForEach-Object {
+Write-Verbose "Searching for templates to package..."
+$templates = Get-ChildItem -Path "$PSScriptRoot/../templates/*", "$PSScriptRoot/../optimization-engine*" -Directory -ErrorAction SilentlyContinue
+
+if ($templates)
+{
+ Write-Verbose "Found $($templates.Count) template(s) to process"
+}
+else
+{
+ Write-Verbose "No templates found to package"
+}
+
+$templates | ForEach-Object {
$srcDir = $_
$templateName = $srcDir.Name
# Skip if not the specified template
if ($Template -ne "*" -and $Template -ne $templateName)
{
+ Write-Verbose "Skipping template '$templateName' (doesn't match pattern '$Template')"
return
}
Write-Host "Building template $templateName..."
+ Write-Verbose " Processing template: $templateName from $($srcDir.FullName)"
# Get custom build configuration
+ Write-Verbose " Loading build configuration from: $($srcDir.FullName)/.build.config"
$buildConfig = Get-Content "$_/.build.config" -ErrorAction SilentlyContinue | ConvertFrom-Json -Depth 10
# Backfill config options to avoid null references
@@ -110,59 +169,89 @@ Get-ChildItem -Path "$PSScriptRoot/../templates/*", "$PSScriptRoot/../optimizati
# Create target directory
$destDir = "$outdir/$templateName"
+ Write-Verbose " Creating target directory: $destDir"
Remove-Item $destDir -Recurse -ErrorAction SilentlyContinue
- & "$PSScriptRoot/New-Directory" $destDir
+ & "$PSScriptRoot/New-Directory.ps1" $destDir
# Copy required files
Write-Host " Copying files..."
- Get-ChildItem $srcDir | Copy-Item -Destination $destDir -Recurse -Exclude ".build.config,.buildignore,scaffold.json"
+ $sourceFiles = Get-ChildItem $srcDir | Where-Object { $_.Name -notin @(".build.config", ".buildignore", "scaffold.json") }
+ Write-Verbose " Copying $($sourceFiles.Count) items from source to destination"
+ $sourceFiles | Copy-Item -Destination $destDir -Recurse
# Remove ignored files
$ignoredFiles = (Get-Content "$srcDir/.buildignore" -ErrorAction SilentlyContinue) + $buildConfig.ignore
if ($ignoredFiles.Length)
{
Write-Host " Removing ignored files..."
- $ignoredFiles `
- | ForEach-Object {
+ Write-Verbose " Processing $($ignoredFiles.Length) ignore pattern(s)"
+ $removedCount = 0
+ $ignoredFiles | ForEach-Object {
$file = $_
if (Test-Path "$destDir/$file")
{
- Write-Verbose "Removing $file"
+ Write-Verbose " Removing: $file"
Remove-Item "$destDir/$file" -Recurse -Force
+ $removedCount++
}
}
+ Write-Verbose " Removed $removedCount ignored file(s)"
+ }
+ else
+ {
+ Write-Verbose " No files to ignore"
}
# Combine KQL files, if specified
if ($buildConfig.combineKql.Length)
{
Write-Host " Combining KQL files..."
+ Write-Verbose " Processing $($buildConfig.combineKql.Length) KQL combination(s)"
$buildConfig.combineKql | ForEach-Object {
+ Write-Verbose " Combining $($_.files.Length) files into $($_.name)"
$combinedScript = ".execute database script with (ContinueOnErrors=true)`n<|`n//`n"
$_.files | ForEach-Object {
+ Write-Verbose " Including file: $_"
$combinedScript += Get-Content "$srcDir/$_" -Raw
}
$combinedScript = $combinedScript -replace '(\r?\n)(\r?\n)+', '$1//$1'
- $combinedScript | Out-File "$destDir/../$($_.name)" -Encoding utf8 -Force
- Write-Verbose "Combined $($_.files.Length) files into $($_.name)"
+ $outputPath = "$destDir/../$($_.name)"
+ Write-Verbose " Writing combined KQL to: $outputPath"
+ $combinedScript | Out-File $outputPath -Encoding utf8 -Force
}
}
+ else
+ {
+ Write-Verbose " No KQL files to combine"
+ }
# Update placeholder variables
if ($buildConfig.variableExpansion.Length)
{
Write-Host " Expanding variables..."
+ Write-Verbose " Processing $($buildConfig.variableExpansion.Length) file(s) for variable expansion"
+ $expandedCount = 0
$buildConfig.variableExpansion | ForEach-Object {
if (Test-Path "$destDir/$_")
{
- Write-Verbose "Updating $_"
+ Write-Verbose " Expanding variables in: $_"
(Get-Content "$destDir/$_" -Raw) `
-replace '\$\$ftkver\$\$', $ver `
-replace '\$\$build-date\$\$', (Get-Date -Format 'yyyy-MM-dd') `
-replace '\$\$build-month\$\$', (Get-Date -Format 'MMMM yyyy') `
| Out-File "$destDir/$_" -Encoding utf8 -Force
+ $expandedCount++
+ }
+ else
+ {
+ Write-Verbose " File not found for variable expansion: $_"
}
}
+ Write-Verbose " Expanded variables in $expandedCount file(s)"
+ }
+ else
+ {
+ Write-Verbose " No variable expansion configured"
}
# Move files, if specified
@@ -184,11 +273,21 @@ Get-ChildItem -Path "$PSScriptRoot/../templates/*", "$PSScriptRoot/../optimizati
# Build main.bicep, if applicable
if (Test-Path "$srcDir/main.bicep")
{
+ Write-Verbose " Found main.bicep, generating deployment files"
Build-MainBicep $destDir
}
+ else
+ {
+ Write-Verbose " No main.bicep found, skipping deployment file generation"
+ }
# Update version in ftkver.txt files
- Get-ChildItem $destDir -Include ftkver.txt -Recurse | ForEach-Object { $ver | Out-File $_ -NoNewline }
+ $versionFiles = Get-ChildItem $destDir -Include ftkver.txt -Recurse
+ Write-Verbose " Updating $($versionFiles.Count) version file(s) with version: $ver"
+ $versionFiles | ForEach-Object {
+ Write-Verbose " Updating version in: $($_.FullName)"
+ $ver | Out-File $_ -NoNewline
+ }
Write-Host ''
}
diff --git a/src/scripts/Deploy-Toolkit.ps1 b/src/scripts/Deploy-Toolkit.ps1
index bf2828a94..371c088f6 100644
--- a/src/scripts/Deploy-Toolkit.ps1
+++ b/src/scripts/Deploy-Toolkit.ps1
@@ -78,9 +78,10 @@ if ($Test -and $Demo)
# Generates a unique name based on the signed in username and computer name for local testing
function Get-UniqueName()
{
+ # Cross-platform name detection (PowerShell 7+)
# NOTE: For some reason, using variables directly does not get the value until we write them
- $c = $env:ComputerName
- $u = $env:USERNAME
+ $u = $env:USERNAME ?? $env:USER ?? "unknown"
+ $c = ($env:COMPUTERNAME ?? $env:HOSTNAME ?? "local").Trim()
$c | Out-Null
$u | Out-Null
return "ftk-$u-$c".ToLower()
@@ -104,13 +105,27 @@ if (Test-Path "$PSScriptRoot/../workbooks/$Template")
# Find bicep file
# NOTE: Include templates after release to account for test templates, which are not included in release builds
-@("$PSScriptRoot/../../release") `
-| ForEach-Object { Get-Item (Join-Path $_ $Template (iff $Test test/main.test.bicep main.bicep)) -ErrorAction SilentlyContinue } `
+# Resolve template file candidates first to avoid silent exit when none are found
+$templateFileCandidates = @("$PSScriptRoot/../../release") `
+| ForEach-Object { Get-Item (Join-Path $_ $Template (iff $Test test/main.test.bicep main.bicep)) -ErrorAction SilentlyContinue }
+if (-not $templateFileCandidates)
+{
+ Write-Error "Template '$Template' not found under '$PSScriptRoot/../../release'."
+ return
+}
+
+$templateFileCandidates `
| ForEach-Object {
$templateFile = $_
$templateName = iff $Test ($templateFile.Directory.Parent.Name + "/test") $templateFile.Directory.Name
$parentFolder = iff $Test $templateFile.Directory.Parent.Parent.Name $templateFile.Directory.Parent.Name
- $targetScope = (Get-Content $templateFile | Select-String "targetScope = '([^']+)'").Matches[0].Captures[0].Groups[1].Value
+ $tsMatch = (Get-Content $templateFile | Select-String "targetScope = '([^']+)'")
+ if ($null -eq $tsMatch -or $tsMatch.Matches.Count -eq 0)
+ {
+ Write-Error "Could not determine targetScope in $($templateFile.FullName). Expected: targetScope = 'resourceGroup'|'subscription'|'tenant'."
+ return
+ }
+ $targetScope = $tsMatch.Matches[0].Groups[1].Value
# Fall back to default parameters if none were provided
$Parameters = iff ($null -eq $Parameters) $defaultParameters["$templateName$(iff $Demo '/demo' '')"] $Parameters
@@ -145,11 +160,11 @@ if (Test-Path "$PSScriptRoot/../workbooks/$Template")
else
{
# Create resource group if it doesn't exist
- Write-Verbose 'Checking resource group $ResourceGroup...'
+ Write-Verbose "Checking resource group $ResourceGroup..."
$rg = Get-AzResourceGroup $ResourceGroup -ErrorAction SilentlyContinue
if ($null -eq $rg)
{
- Write-Verbose 'Creating resource group $ResourceGroup...'
+ Write-Verbose "Creating resource group $ResourceGroup..."
New-AzResourceGroup `
-Name $ResourceGroup `
-Location $Location `
@@ -205,7 +220,7 @@ if (Test-Path "$PSScriptRoot/../workbooks/$Template")
Write-Verbose 'Starting tenant deployment...'
$azContext = (Get-AzContext).Tenant
- Write-Host " → [tenant] $(iff ([string]::IsNullOrWhitespace($azContext.Name)) $azContext.Id $azContext.Name)..."
+ Write-Host " → [tenant] $(iff ([string]::IsNullOrWhiteSpace($azContext.Name)) $azContext.Id $azContext.Name)..."
$Parameters.Keys | ForEach-Object { Write-Host " $($_) = $($Parameters[$_])" }
if ($Debug)
diff --git a/src/templates/finops-hub/.build.config b/src/templates/finops-hub/.build.config
index 0a7ea8c24..11a298117 100644
--- a/src/templates/finops-hub/.build.config
+++ b/src/templates/finops-hub/.build.config
@@ -20,27 +20,27 @@
{
"name": "finops-hub-fabric-setup-Ingestion.kql",
"files": [
- "modules/scripts/OpenDataFunctions_resource_type_1.kql",
- "modules/scripts/OpenDataFunctions_resource_type_2.kql",
- "modules/scripts/OpenDataFunctions_resource_type_3.kql",
- "modules/scripts/OpenDataFunctions_resource_type_4.kql",
- "modules/scripts/OpenDataFunctions_resource_type_5.kql",
- "modules/scripts/OpenDataFunctions.kql",
- "modules/scripts/Common.kql",
- "modules/scripts/IngestionSetup_HubInfra.kql",
- "modules/scripts/IngestionSetup_RawTables.kql",
- "modules/scripts/IngestionSetup_v1_0.kql",
- "modules/scripts/IngestionSetup_v1_2.kql"
+ "modules/Microsoft.FinOpsHubs/Analytics/scripts/OpenDataFunctions_resource_type_1.kql",
+ "modules/Microsoft.FinOpsHubs/Analytics/scripts/OpenDataFunctions_resource_type_2.kql",
+ "modules/Microsoft.FinOpsHubs/Analytics/scripts/OpenDataFunctions_resource_type_3.kql",
+ "modules/Microsoft.FinOpsHubs/Analytics/scripts/OpenDataFunctions_resource_type_4.kql",
+ "modules/Microsoft.FinOpsHubs/Analytics/scripts/OpenDataFunctions_resource_type_5.kql",
+ "modules/Microsoft.FinOpsHubs/Analytics/scripts/OpenDataFunctions.kql",
+ "modules/Microsoft.FinOpsHubs/Analytics/scripts/Common.kql",
+ "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_HubInfra.kql",
+ "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql",
+ "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_0.kql",
+ "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql"
]
},
{
"name": "finops-hub-fabric-setup-Hub.kql",
"files": [
- "modules/scripts/Common.kql",
- "modules/scripts/HubSetup_OpenData.kql",
- "modules/scripts/HubSetup_v1_0.kql",
- "modules/scripts/HubSetup_v1_2.kql",
- "modules/scripts/HubSetup_Latest.kql"
+ "modules/Microsoft.FinOpsHubs/Analytics/scripts/Common.kql",
+ "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_OpenData.kql",
+ "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_0.kql",
+ "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_2.kql",
+ "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql"
]
}
]
diff --git a/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/app.bicep b/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/app.bicep
new file mode 100644
index 000000000..e020a8031
--- /dev/null
+++ b/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/app.bicep
@@ -0,0 +1,1720 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { finOpsToolkitVersion, HubAppProperties } from '../../fx/hub-types.bicep'
+
+
+//==============================================================================
+// Parameters
+//==============================================================================
+
+@description('Required. FinOps hub app getting deployed.')
+param app HubAppProperties
+
+
+//==============================================================================
+// Variables
+//==============================================================================
+
+var CONFIG = 'config'
+var INGESTION = 'ingestion'
+var MSEXPORTS = 'msexports'
+
+// Separator used to separate ingestion ID from file name for ingested files
+var ingestionIdFileNameSeparator = '__'
+
+
+//==============================================================================
+// Resources
+//==============================================================================
+
+// Register app
+module appRegistration '../../fx/hub-app.bicep' = {
+ name: 'Microsoft.CostManagement.Exports_Register'
+ params: {
+ app: app
+ version: finOpsToolkitVersion
+ features: [
+ 'Storage' // msexports + schema files
+ 'DataFactory' // Move files from msexports to ingestion
+ ]
+ storageRoles: [
+ // User Access Administrator -- https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#user-access-administrator
+ // Used to create Cost Management exports (which require access to grant access)
+ '18d7d88d-d35e-4fb5-a5c3-7773c20a72d9'
+ ]
+ }
+}
+
+//------------------------------------------------------------------------------
+// Storage
+//------------------------------------------------------------------------------
+
+// Upload schema files
+module schemaFiles '../../fx/hub-storage.bicep' = {
+ name: 'Microsoft.CostManagement.Exports_Storage.SchemaFiles'
+ dependsOn: [
+ appRegistration
+ ]
+ params: {
+ app: app
+ container: 'config'
+ files: {
+ // cSpell:ignore actualcost, amortizedcost, focuscost, pricesheet, reservationdetails, reservationrecommendations, reservationtransactions
+ 'schemas/actualcost_c360-2025-04.json': loadTextContent('./schemas/actualcost_c360-2025-04.json')
+ 'schemas/amortizedcost_c360-2025-04.json': loadTextContent('./schemas/amortizedcost_c360-2025-04.json')
+ 'schemas/focuscost_1.2.json': loadTextContent('./schemas/focuscost_1.2.json')
+ 'schemas/focuscost_1.2-preview.json': loadTextContent('./schemas/focuscost_1.2-preview.json')
+ 'schemas/focuscost_1.0r2.json': loadTextContent('./schemas/focuscost_1.0r2.json')
+ 'schemas/focuscost_1.0.json': loadTextContent('./schemas/focuscost_1.0.json')
+ 'schemas/focuscost_1.0-preview(v1).json': loadTextContent('./schemas/focuscost_1.0-preview(v1).json')
+ 'schemas/pricesheet_2023-05-01_ea.json': loadTextContent('./schemas/pricesheet_2023-05-01_ea.json')
+ 'schemas/pricesheet_2023-05-01_mca.json': loadTextContent('./schemas/pricesheet_2023-05-01_mca.json')
+ 'schemas/reservationdetails_2023-03-01.json': loadTextContent('./schemas/reservationdetails_2023-03-01.json')
+ 'schemas/reservationrecommendations_2023-05-01_ea.json': loadTextContent('./schemas/reservationrecommendations_2023-05-01_ea.json')
+ 'schemas/reservationrecommendations_2023-05-01_mca.json': loadTextContent('./schemas/reservationrecommendations_2023-05-01_mca.json')
+ 'schemas/reservationtransactions_2023-05-01_ea.json': loadTextContent('./schemas/reservationtransactions_2023-05-01_ea.json')
+ 'schemas/reservationtransactions_2023-05-01_mca.json': loadTextContent('./schemas/reservationtransactions_2023-05-01_mca.json')
+ }
+ }
+}
+
+// Create msexports container
+module exportContainer '../../fx/hub-storage.bicep' = {
+ name: 'Microsoft.CostManagement.Exports_Storage.ExportContainer'
+ dependsOn: [
+ appRegistration
+ ]
+ params: {
+ app: app
+ container: MSEXPORTS
+ }
+}
+
+//------------------------------------------------------------------------------
+// Data Factory
+//------------------------------------------------------------------------------
+
+resource dataFactory 'Microsoft.DataFactory/factories@2018-06-01' existing = {
+ name: app.dataFactory
+ dependsOn: [
+ appRegistration
+ ]
+
+ // cSpell:ignore linkedservices
+ resource linkedService_storageAccount 'linkedservices' existing = {
+ name: app.storage
+ }
+
+ resource dataset_config 'datasets' existing = {
+ name: CONFIG
+ }
+
+ resource dataset_ingestion 'datasets' existing = {
+ name: INGESTION
+ }
+
+ resource dataset_ingestion_files 'datasets' existing = {
+ name: '${INGESTION}_files'
+ }
+
+ resource dataset_manifest 'datasets' = {
+ name: 'manifest'
+ properties: {
+ parameters: {
+ fileName: {
+ type: 'String'
+ defaultValue: 'manifest.json'
+ }
+ folderPath: {
+ type: 'String'
+ defaultValue: MSEXPORTS
+ }
+ }
+ type: 'Json'
+ typeProperties: {
+ location: {
+ type: 'AzureBlobFSLocation'
+ fileName: {
+ value: '@{dataset().fileName}'
+ type: 'Expression'
+ }
+ folderPath: {
+ value: '@{dataset().folderPath}'
+ type: 'Expression'
+ }
+ }
+ }
+ linkedServiceName: {
+ // TODO: Should linked service names/references be part of settings? Should datasets be hub modules?
+ referenceName: app.storage
+ type: 'LinkedServiceReference'
+ }
+ }
+ }
+
+ resource dataset_msexports 'datasets' = {
+ name: replace('${MSEXPORTS}', '-', '_')
+ properties: {
+ parameters: {
+ blobPath: {
+ type: 'String'
+ }
+ }
+ type: 'DelimitedText'
+ typeProperties: {
+ location: {
+ type: 'AzureBlobFSLocation'
+ fileName: {
+ value: '@{dataset().blobPath}'
+ type: 'Expression'
+ }
+ fileSystem: exportContainer.outputs.containerName
+ }
+ columnDelimiter: ','
+ escapeChar: '"'
+ quoteChar: '"'
+ firstRowAsHeader: true
+ }
+ linkedServiceName: {
+ referenceName: linkedService_storageAccount.name
+ type: 'LinkedServiceReference'
+ }
+ }
+ }
+
+ resource dataset_msexports_gzip 'datasets' = {
+ name: '${MSEXPORTS}_gzip'
+ properties: {
+ parameters: {
+ blobPath: {
+ type: 'String'
+ }
+ }
+ type: 'DelimitedText'
+ typeProperties: {
+ location: {
+ type: 'AzureBlobFSLocation'
+ fileName: {
+ value: '@{dataset().blobPath}'
+ type: 'Expression'
+ }
+ fileSystem: MSEXPORTS
+ }
+ columnDelimiter: ','
+ escapeChar: '"'
+ quoteChar: '"'
+ firstRowAsHeader: true
+ compressionCodec: 'Gzip'
+ }
+ linkedServiceName: {
+ referenceName: linkedService_storageAccount.name
+ type: 'LinkedServiceReference'
+ }
+ }
+ }
+
+ resource dataset_msexports_parquet 'datasets' = {
+ name: '${MSEXPORTS}_parquet'
+ properties: {
+ parameters: {
+ blobPath: {
+ type: 'String'
+ }
+ }
+ type: 'Parquet'
+ typeProperties: {
+ location: {
+ type: 'AzureBlobFSLocation'
+ fileName: {
+ value: '@{dataset().blobPath}'
+ type: 'Expression'
+ }
+ fileSystem: MSEXPORTS
+ }
+ }
+ linkedServiceName: {
+ referenceName: linkedService_storageAccount.name
+ type: 'LinkedServiceReference'
+ }
+ }
+ }
+
+ //---------------------------------------------------------------------------
+ // msexports_ExecuteETL pipeline
+ // Triggered by msexports_ManifestAdded trigger
+ //---------------------------------------------------------------------------
+ resource pipeline_ExecuteExportsETL 'pipelines' = {
+ name: '${MSEXPORTS}_ExecuteETL'
+ properties: {
+ activities: [
+ { // Wait
+ name: 'Wait'
+ description: 'Files may not be available immediately after being created.'
+ type: 'Wait'
+ dependsOn: []
+ userProperties: []
+ typeProperties: {
+ waitTimeInSeconds: 60
+ }
+ }
+ { // Read Manifest
+ name: 'Read Manifest'
+ description: 'Load the export manifest to determine the scope, dataset, and date range.'
+ type: 'Lookup'
+ dependsOn: [
+ {
+ activity: 'Wait'
+ dependencyConditions: ['Completed']
+ }
+ ]
+ policy: {
+ timeout: '0.12:00:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ source: {
+ type: 'JsonSource'
+ storeSettings: {
+ type: 'AzureBlobFSReadSettings'
+ recursive: true
+ enablePartitionDiscovery: false
+ }
+ formatSettings: {
+ type: 'JsonReadSettings'
+ }
+ }
+ dataset: {
+ referenceName: dataFactory::dataset_manifest.name
+ type: 'DatasetReference'
+ parameters: {
+ fileName: {
+ value: '@pipeline().parameters.fileName'
+ type: 'Expression'
+ }
+ folderPath: {
+ value: '@pipeline().parameters.folderPath'
+ type: 'Expression'
+ }
+ }
+ }
+ }
+ }
+ { // Set Has No Rows
+ name: 'Set Has No Rows'
+ description: 'Check the row count '
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Read Manifest'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'hasNoRows'
+ value: {
+ value: '@or(equals(activity(\'Read Manifest\').output.firstRow.blobCount, null), equals(activity(\'Read Manifest\').output.firstRow.blobCount, 0))'
+ type: 'Expression'
+ }
+ }
+ }
+ { // Set Export Dataset Type
+ name: 'Set Export Dataset Type'
+ description: 'Save the dataset type from the export manifest.'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Read Manifest'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'exportDatasetType'
+ value: {
+ value: '@activity(\'Read Manifest\').output.firstRow.exportConfig.type'
+ type: 'Expression'
+ }
+ }
+ }
+ { // Set MCA Column
+ name: 'Set MCA Column'
+ description: 'Determines if the dataset schema has channel-specific columns and saves the column name that only exists in MCA to determine if it is an MCA dataset.'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Set Export Dataset Type'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'mcaColumnToCheck'
+ value: {
+ // cSpell:ignore pricesheet, reservationtransactions, reservationrecommendations
+ value: '@if(contains(createArray(\'pricesheet\', \'reservationtransactions\'), toLower(variables(\'exportDatasetType\'))), \'BillingProfileId\', if(equals(toLower(variables(\'exportDatasetType\')), \'reservationrecommendations\'), \'Net Savings\', null))'
+ type: 'Expression'
+ }
+ }
+ }
+ { // Set Export Dataset Version
+ name: 'Set Export Dataset Version'
+ description: 'Save the dataset version from the export manifest.'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Read Manifest'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'exportDatasetVersion'
+ value: {
+ value: '@activity(\'Read Manifest\').output.firstRow.exportConfig.dataVersion'
+ type: 'Expression'
+ }
+ }
+ }
+ { // Detect Channel
+ name: 'Detect Channel'
+ description: 'Determines what channel this export is from. Switch statement handles the different file types if the mcaColumnToCheck variable is set.'
+ type: 'Switch'
+ dependsOn: [
+ {
+ activity: 'Set Has No Rows'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ {
+ activity: 'Set MCA Column'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ {
+ activity: 'Set Export Dataset Version'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ on: {
+ value: '@if(or(empty(variables(\'mcaColumnToCheck\')), variables(\'hasNoRows\')), \'ignore\', last(array(split(activity(\'Read Manifest\').output.firstRow.blobs[0].blobName, \'.\'))))'
+ type: 'Expression'
+ }
+ cases: [
+ { // csv
+ value: 'csv'
+ activities: [
+ {
+ name: 'Check for MCA Column in CSV'
+ description: 'Checks the dataset to determine if the applicable MCA-specific column exists.'
+ type: 'Lookup'
+ dependsOn: []
+ policy: {
+ timeout: '0.12:00:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ source: {
+ type: 'DelimitedTextSource'
+ storeSettings: {
+ type: 'AzureBlobFSReadSettings'
+ recursive: false
+ enablePartitionDiscovery: false
+ }
+ formatSettings: {
+ type: 'DelimitedTextReadSettings'
+ }
+ }
+ dataset: {
+ referenceName: dataFactory::dataset_msexports.name
+ type: 'DatasetReference'
+ parameters: {
+ blobPath: {
+ value: '@activity(\'Read Manifest\').output.firstRow.blobs[0].blobName'
+ type: 'Expression'
+ }
+ }
+ }
+ }
+ }
+ {
+ name: 'Set Schema File with Channel in CSV'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Check for MCA Column in CSV'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'schemaFile'
+ value: {
+ value: '@toLower(concat(variables(\'exportDatasetType\'), \'_\', variables(\'exportDatasetVersion\'), if(and(contains(activity(\'Check for MCA Column in CSV\').output, \'firstRow\'), contains(activity(\'Check for MCA Column in CSV\').output.firstRow, variables(\'mcaColumnToCheck\'))), \'_mca\', \'_ea\'), \'.json\'))'
+ type: 'Expression'
+ }
+ }
+ }
+ ]
+ }
+ { // gz
+ value: 'gz'
+ activities: [
+ {
+ name: 'Check for MCA Column in Gzip CSV'
+ description: 'Checks the dataset to determine if the applicable MCA-specific column exists.'
+ type: 'Lookup'
+ dependsOn: []
+ policy: {
+ timeout: '0.12:00:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ source: {
+ type: 'DelimitedTextSource'
+ storeSettings: {
+ type: 'AzureBlobFSReadSettings'
+ recursive: false
+ enablePartitionDiscovery: false
+ }
+ formatSettings: {
+ type: 'DelimitedTextReadSettings'
+ }
+ }
+ dataset: {
+ referenceName: dataFactory::dataset_msexports_gzip.name
+ type: 'DatasetReference'
+ parameters: {
+ blobPath: {
+ value: '@activity(\'Read Manifest\').output.firstRow.blobs[0].blobName'
+ type: 'Expression'
+ }
+ }
+ }
+ }
+ }
+ {
+ name: 'Set Schema File with Channel in Gzip CSV'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Check for MCA Column in Gzip CSV'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'schemaFile'
+ value: {
+ value: '@toLower(concat(variables(\'exportDatasetType\'), \'_\', variables(\'exportDatasetVersion\'), if(and(contains(activity(\'Check for MCA Column in Gzip CSV\').output, \'firstRow\'), contains(activity(\'Check for MCA Column in Gzip CSV\').output.firstRow, variables(\'mcaColumnToCheck\'))), \'_mca\', \'_ea\'), \'.json\'))'
+ type: 'Expression'
+ }
+ }
+ }
+ ]
+ }
+ { // parquet
+ value: 'parquet'
+ activities: [
+ {
+ name: 'Check for MCA Column in Parquet'
+ description: 'Checks the dataset to determine if the applicable MCA-specific column exists.'
+ type: 'Lookup'
+ dependsOn: []
+ policy: {
+ timeout: '0.12:00:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ source: {
+ type: 'ParquetSource'
+ storeSettings: {
+ type: 'AzureBlobFSReadSettings'
+ recursive: false
+ enablePartitionDiscovery: false
+ }
+ formatSettings: {
+ type: 'ParquetReadSettings'
+ }
+ }
+ dataset: {
+ referenceName: dataFactory::dataset_msexports_parquet.name
+ type: 'DatasetReference'
+ parameters: {
+ blobPath: {
+ value: '@activity(\'Read Manifest\').output.firstRow.blobs[0].blobName'
+ type: 'Expression'
+ }
+ }
+ }
+ }
+ }
+ {
+ name: 'Set Schema File with Channel for Parquet'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Check for MCA Column in Parquet'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'schemaFile'
+ value: {
+ value: '@toLower(concat(variables(\'exportDatasetType\'), \'_\', variables(\'exportDatasetVersion\'), if(and(contains(activity(\'Check for MCA Column in Parquet\').output, \'firstRow\'), contains(activity(\'Check for MCA Column in Parquet\').output.firstRow, variables(\'mcaColumnToCheck\'))), \'_mca\', \'_ea\'), \'.json\'))'
+ type: 'Expression'
+ }
+ }
+ }
+ ]
+ }
+ ]
+ defaultActivities: [
+ {
+ name: 'Set Schema File'
+ type: 'SetVariable'
+ dependsOn: []
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'schemaFile'
+ value: {
+ value: '@toLower(concat(variables(\'exportDatasetType\'), \'_\', variables(\'exportDatasetVersion\'), \'.json\'))'
+ type: 'Expression'
+ }
+ }
+ }
+ ]
+ }
+ }
+ { // Set Scope
+ name: 'Set Scope'
+ description: 'Save the scope from the export manifest.'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Read Manifest'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'scope'
+ value: {
+ value: '@split(toLower(activity(\'Read Manifest\').output.firstRow.exportConfig.resourceId), \'/providers/microsoft.costmanagement/exports/\')[0]'
+ type: 'Expression'
+ }
+ }
+ }
+ { // Set Date
+ name: 'Set Date'
+ description: 'Save the exported month from the export manifest.'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Read Manifest'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'date'
+ value: {
+ value: '@replace(substring(activity(\'Read Manifest\').output.firstRow.runInfo.startDate, 0, 7), \'-\', \'\')'
+ type: 'Expression'
+ }
+ }
+ }
+ { // Error: ManifestReadFailed
+ name: 'Failed to Read Manifest'
+ type: 'Fail'
+ dependsOn: [
+ {
+ activity: 'Set Date'
+ dependencyConditions: ['Failed']
+ }
+ {
+ activity: 'Set Export Dataset Type'
+ dependencyConditions: ['Failed']
+ }
+ {
+ activity: 'Set Scope'
+ dependencyConditions: ['Failed']
+ }
+ {
+ activity: 'Read Manifest'
+ dependencyConditions: ['Failed']
+ }
+ {
+ activity: 'Set Export Dataset Version'
+ dependencyConditions: ['Failed']
+ }
+ {
+ activity: 'Detect Channel'
+ dependencyConditions: ['Failed']
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ message: {
+ value: '@concat(\'Failed to read the manifest file for this export run. Manifest path: \', pipeline().parameters.folderPath)'
+ type: 'Expression'
+ }
+ errorCode: 'ManifestReadFailed'
+ }
+ }
+ { // Check Schema
+ name: 'Check Schema'
+ description: 'Verify that the schema file exists in storage.'
+ type: 'GetMetadata'
+ dependsOn: [
+ {
+ activity: 'Set Scope'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ {
+ activity: 'Set Date'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ {
+ activity: 'Detect Channel'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ timeout: '0.12:00:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ dataset: {
+ referenceName: dataFactory::dataset_config.name
+ type: 'DatasetReference'
+ parameters: {
+ fileName: {
+ value: '@variables(\'schemaFile\')'
+ type: 'Expression'
+ }
+ folderPath: '${schemaFiles.outputs.containerName}/schemas'
+ }
+ }
+ fieldList: ['exists']
+ storeSettings: {
+ type: 'AzureBlobFSReadSettings'
+ recursive: true
+ enablePartitionDiscovery: false
+ }
+ formatSettings: {
+ type: 'JsonReadSettings'
+ }
+ }
+ }
+ { // Error: SchemaNotFound
+ name: 'Schema Not Found'
+ type: 'Fail'
+ dependsOn: [
+ {
+ activity: 'Check Schema'
+ dependencyConditions: ['Failed']
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ message: {
+ value: '@concat(\'The \', variables(\'schemaFile\'), \' schema mapping file was not found. Please confirm version \', variables(\'exportDatasetVersion\'), \' of the \', variables(\'exportDatasetType\'), \' dataset is supported by this version of FinOps hubs. You may need to upgrade to a newer release. To add support for another dataset, you can create a custom mapping file.\')'
+ type: 'Expression'
+ }
+ errorCode: 'SchemaNotFound'
+ }
+ }
+ { // Set Hub Dataset
+ name: 'Set Hub Dataset'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Set Export Dataset Type'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'hubDataset'
+ value: {
+ value: '@if(equals(toLower(variables(\'exportDatasetType\')), \'focuscost\'), \'Costs\', if(equals(toLower(variables(\'exportDatasetType\')), \'pricesheet\'), \'Prices\', if(equals(toLower(variables(\'exportDatasetType\')), \'reservationdetails\'), \'CommitmentDiscountUsage\', if(equals(toLower(variables(\'exportDatasetType\')), \'reservationrecommendations\'), \'Recommendations\', if(equals(toLower(variables(\'exportDatasetType\')), \'reservationtransactions\'), \'Transactions\', if(equals(toLower(variables(\'exportDatasetType\')), \'actualcost\'), \'ActualCosts\', if(equals(toLower(variables(\'exportDatasetType\')), \'amortizedcost\'), \'AmortizedCosts\', toLower(variables(\'exportDatasetType\')))))))))'
+ type: 'Expression'
+ }
+ }
+ }
+ { // Set Destination Folder
+ name: 'Set Destination Folder'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Check Schema'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ {
+ activity: 'Set Hub Dataset'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'destinationFolder'
+ value: {
+ value: '@replace(concat(variables(\'hubDataset\'),\'/\',substring(variables(\'date\'), 0, 4),\'/\',substring(variables(\'date\'), 4, 2),\'/\',toLower(variables(\'scope\')), if(equals(variables(\'hubDataset\'), \'Recommendations\'), activity(\'Read Manifest\').output.firstRow.exportConfig.exportName, \'\')),\'//\',\'/\')'
+ type: 'Expression'
+ }
+ }
+ }
+ { // For Each Blob
+ name: 'For Each Blob'
+ description: 'Loop thru each exported file listed in the manifest.'
+ type: 'ForEach'
+ dependsOn: [
+ {
+ activity: 'Set Destination Folder'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ items: {
+ value: '@if(variables(\'hasNoRows\'), json(\'[]\'), activity(\'Read Manifest\').output.firstRow.blobs)'
+ type: 'Expression'
+ }
+ batchCount: app.hub.options.privateRouting ? 4 : 30 // so we don't overload the managed runtime
+ isSequential: false
+ activities: [
+ { // Execute
+ name: 'Execute'
+ description: 'Run the ingestion ETL pipeline.'
+ type: 'ExecutePipeline'
+ dependsOn: []
+ policy: {
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ pipeline: {
+ referenceName: pipeline_ToIngestion.name
+ type: 'PipelineReference'
+ }
+ waitOnCompletion: true
+ parameters: {
+ blobPath: {
+ value: '@item().blobName'
+ type: 'Expression'
+ }
+ destinationFolder: {
+ value: '@variables(\'destinationFolder\')'
+ type: 'Expression'
+ }
+ destinationFile: {
+ value: '@last(array(split(replace(replace(item().blobName, \'.gz\', \'\'), \'.csv\', \'.parquet\'), \'/\')))'
+ type: 'Expression'
+ }
+ ingestionId: {
+ value: '@activity(\'Read Manifest\').output.firstRow.runInfo.runId'
+ type: 'Expression'
+ }
+ schemaFile: {
+ value: '@variables(\'schemaFile\')'
+ type: 'Expression'
+ }
+ exportDatasetType: {
+ value: '@variables(\'exportDatasetType\')'
+ type: 'Expression'
+ }
+ exportDatasetVersion: {
+ value: '@variables(\'exportDatasetVersion\')'
+ type: 'Expression'
+ }
+ }
+ }
+ }
+ ]
+ }
+ }
+ { // Copy Manifest
+ name: 'Copy Manifest'
+ description: 'Copy the manifest to the ingestion container to trigger ADX ingestion'
+ type: 'Copy'
+ dependsOn: [
+ {
+ activity: 'For Each Blob'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ timeout: '0.12:00:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ source: {
+ type: 'JsonSource'
+ storeSettings: {
+ type: 'AzureBlobFSReadSettings'
+ recursive: true
+ enablePartitionDiscovery: false
+ }
+ formatSettings: {
+ type: 'JsonReadSettings'
+ }
+ }
+ sink: {
+ type: 'JsonSink'
+ storeSettings: {
+ type: 'AzureBlobFSWriteSettings'
+ }
+ formatSettings: {
+ type: 'JsonWriteSettings'
+ }
+ }
+ enableStaging: false
+ }
+ inputs: [
+ {
+ referenceName: dataFactory::dataset_manifest.name
+ type: 'DatasetReference'
+ parameters: {
+ fileName: 'manifest.json'
+ folderPath: {
+ value: '@pipeline().parameters.folderPath'
+ type: 'Expression'
+ }
+ }
+ }
+ ]
+ outputs: [
+ {
+ referenceName: dataFactory::dataset_manifest.name
+ type: 'DatasetReference'
+ parameters: {
+ fileName: 'manifest.json'
+ folderPath: {
+ value: '@concat(\'${INGESTION}/\', variables(\'destinationFolder\'))'
+ type: 'Expression'
+ }
+ }
+ }
+ ]
+ }
+ ]
+ parameters: {
+ folderPath: {
+ type: 'string'
+ }
+ fileName: {
+ type: 'string'
+ }
+ }
+ variables: {
+ date: {
+ type: 'String'
+ }
+ destinationFolder: {
+ type: 'String'
+ }
+ exportDatasetType: {
+ type: 'String'
+ }
+ exportDatasetVersion: {
+ type: 'String'
+ }
+ hasNoRows: {
+ type: 'Boolean'
+ }
+ hubDataset: {
+ type: 'String'
+ }
+ mcaColumnToCheck: {
+ type: 'String'
+ }
+ schemaFile: {
+ type: 'String'
+ }
+ scope: {
+ type: 'String'
+ }
+ }
+ annotations: [
+ 'New export'
+ ]
+ }
+ }
+
+ //---------------------------------------------------------------------------
+ // msexports_ETL_ingestion pipeline
+ // Triggered by msexports_ExecuteETL
+ //---------------------------------------------------------------------------
+ resource pipeline_ToIngestion 'pipelines' = {
+ name: '${MSEXPORTS}_ETL_${INGESTION}'
+ properties: {
+ activities: [
+ { // Get Existing Parquet Files
+ name: 'Get Existing Parquet Files'
+ description: 'Get the previously ingested files so we can remove any older data. This is necessary to avoid data duplication in reports.'
+ type: 'GetMetadata'
+ dependsOn: []
+ policy: {
+ timeout: '0.12:00:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ dataset: {
+ referenceName: dataFactory::dataset_ingestion_files.name
+ type: 'DatasetReference'
+ parameters: {
+ folderPath: '@pipeline().parameters.destinationFolder'
+ }
+ }
+ fieldList: [
+ 'childItems'
+ ]
+ storeSettings: {
+ type: 'AzureBlobFSReadSettings'
+ enablePartitionDiscovery: false
+ }
+ formatSettings: {
+ type: 'ParquetReadSettings'
+ }
+ }
+ }
+ { // Filter Out Current Exports
+ name: 'Filter Out Current Exports'
+ description: 'Remove existing files from the current export so those files do not get deleted.'
+ type: 'Filter'
+ dependsOn: [
+ {
+ activity: 'Get Existing Parquet Files'
+ dependencyConditions: [
+ 'Completed'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ items: {
+ value: '@if(contains(activity(\'Get Existing Parquet Files\').output, \'childItems\'), activity(\'Get Existing Parquet Files\').output.childItems, json(\'[]\'))'
+ type: 'Expression'
+ }
+ condition: {
+ // cSpell:ignore endswith
+ value: '@and(endswith(item().name, \'.parquet\'), not(startswith(item().name, concat(pipeline().parameters.ingestionId, \'${ingestionIdFileNameSeparator}\'))))'
+ type: 'Expression'
+ }
+ }
+ }
+ { // Load Schema Mappings
+ name: 'Load Schema Mappings'
+ description: 'Get schema mapping file to use for the CSV to parquet conversion.'
+ type: 'Lookup'
+ dependsOn: []
+ policy: {
+ timeout: '0.12:00:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ source: {
+ type: 'JsonSource'
+ storeSettings: {
+ type: 'AzureBlobFSReadSettings'
+ recursive: true
+ enablePartitionDiscovery: false
+ }
+ formatSettings: {
+ type: 'JsonReadSettings'
+ }
+ }
+ dataset: {
+ referenceName: dataFactory::dataset_config.name
+ type: 'DatasetReference'
+ parameters: {
+ fileName: {
+ value: '@toLower(pipeline().parameters.schemaFile)'
+ type: 'Expression'
+ }
+ folderPath: '${CONFIG}/schemas'
+ }
+ }
+ }
+ }
+ { // Error: SchemaLoadFailed
+ name: 'Failed to Load Schema'
+ type: 'Fail'
+ dependsOn: [
+ {
+ activity: 'Load Schema Mappings'
+ dependencyConditions: [
+ 'Failed'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ message: {
+ value: '@concat(\'Unable to load the \', pipeline().parameters.schemaFile, \' schema file. Please confirm the schema and version are supported for FinOps hubs ingestion. Unsupported files will remain in the msexports container.\')'
+ type: 'Expression'
+ }
+ errorCode: 'SchemaLoadFailed'
+ }
+ }
+ { // Set Additional Columns
+ name: 'Set Additional Columns'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Load Schema Mappings'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'additionalColumns'
+ value: {
+ value: '@intersection(array(json(concat(\'[{"name":"x_SourceProvider","value":"Microsoft"},{"name":"x_SourceName","value":"Cost Management"},{"name":"x_SourceType","value":"\', pipeline().parameters.exportDatasetVersion, \'"},{"name":"x_SourceVersion","value":"\', pipeline().parameters.exportDatasetVersion, \'"}\'))), activity(\'Load Schema Mappings\').output.firstRow.additionalColumns)'
+ type: 'Expression'
+ }
+ }
+ }
+ { // For Each Old File
+ name: 'For Each Old File'
+ description: 'Loop thru each of the existing files from previous exports.'
+ type: 'ForEach'
+ dependsOn: [
+ {
+ activity: 'Convert to Parquet'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ {
+ activity: 'Filter Out Current Exports'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ items: {
+ value: '@activity(\'Filter Out Current Exports\').output.Value'
+ type: 'Expression'
+ }
+ activities: [
+ { // Delete Old Ingested File
+ name: 'Delete Old Ingested File'
+ description: 'Delete the previously ingested files from older exports.'
+ type: 'Delete'
+ dependsOn: []
+ policy: {
+ timeout: '0.12:00:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ dataset: {
+ referenceName: dataFactory::dataset_ingestion.name
+ type: 'DatasetReference'
+ parameters: {
+ blobPath: {
+ value: '@concat(pipeline().parameters.destinationFolder, \'/\', item().name)'
+ type: 'Expression'
+ }
+ }
+ }
+ enableLogging: false
+ storeSettings: {
+ type: 'AzureBlobFSReadSettings'
+ recursive: false
+ enablePartitionDiscovery: false
+ }
+ }
+ }
+ ]
+ }
+ }
+ { // Set Destination Path
+ name: 'Set Destination Path'
+ type: 'SetVariable'
+ dependsOn: []
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'destinationPath'
+ value: {
+ value: '@concat(pipeline().parameters.destinationFolder, \'/\', pipeline().parameters.ingestionId, \'${ingestionIdFileNameSeparator}\', pipeline().parameters.destinationFile)'
+ type: 'Expression'
+ }
+ }
+ }
+ { // Convert to Parquet
+ name: 'Convert to Parquet'
+ description: 'Convert CSV to parquet and move the file to the ${INGESTION} container.'
+ type: 'Switch'
+ dependsOn: [
+ {
+ activity: 'Set Destination Path'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ {
+ activity: 'Load Schema Mappings'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ {
+ activity: 'Set Additional Columns'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ on: {
+ value: '@last(array(split(pipeline().parameters.blobPath, \'.\')))'
+ type: 'Expression'
+ }
+ cases: [
+ { // CSV
+ value: 'csv'
+ activities: [
+ { // Convert CSV File
+ name: 'Convert CSV File'
+ type: 'Copy'
+ dependsOn: []
+ policy: {
+ timeout: '0.00:10:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ source: {
+ type: 'DelimitedTextSource'
+ additionalColumns: {
+ value: '@variables(\'additionalColumns\')'
+ type: 'Expression'
+ }
+ storeSettings: {
+ type: 'AzureBlobFSReadSettings'
+ recursive: true
+ enablePartitionDiscovery: false
+ }
+ formatSettings: {
+ type: 'DelimitedTextReadSettings'
+ }
+ }
+ sink: {
+ type: 'ParquetSink'
+ storeSettings: {
+ type: 'AzureBlobFSWriteSettings'
+ }
+ formatSettings: {
+ type: 'ParquetWriteSettings'
+ fileExtension: '.parquet'
+ }
+ }
+ enableStaging: false
+ parallelCopies: 1
+ validateDataConsistency: false
+ translator: {
+ value: '@activity(\'Load Schema Mappings\').output.firstRow.translator'
+ type: 'Expression'
+ }
+ }
+ inputs: [
+ {
+ referenceName: dataFactory::dataset_msexports.name
+ type: 'DatasetReference'
+ parameters: {
+ blobPath: {
+ value: '@pipeline().parameters.blobPath'
+ type: 'Expression'
+ }
+ }
+ }
+ ]
+ outputs: [
+ {
+ referenceName: dataFactory::dataset_ingestion.name
+ type: 'DatasetReference'
+ parameters: {
+ blobPath: {
+ value: '@variables(\'destinationPath\')'
+ type: 'Expression'
+ }
+ }
+ }
+ ]
+ }
+ ]
+ }
+ { // GZ
+ value: 'gz'
+ activities: [
+ { // Convert GZip CSV File
+ name: 'Convert GZip CSV File'
+ type: 'Copy'
+ dependsOn: []
+ policy: {
+ timeout: '0.00:10:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ source: {
+ type: 'DelimitedTextSource'
+ additionalColumns: {
+ value: '@variables(\'additionalColumns\')'
+ type: 'Expression'
+ }
+ storeSettings: {
+ type: 'AzureBlobFSReadSettings'
+ recursive: true
+ enablePartitionDiscovery: false
+ }
+ formatSettings: {
+ type: 'DelimitedTextReadSettings'
+ }
+ }
+ sink: {
+ type: 'ParquetSink'
+ storeSettings: {
+ type: 'AzureBlobFSWriteSettings'
+ }
+ formatSettings: {
+ type: 'ParquetWriteSettings'
+ fileExtension: '.parquet'
+ }
+ }
+ enableStaging: false
+ parallelCopies: 1
+ validateDataConsistency: false
+ translator: {
+ value: '@activity(\'Load Schema Mappings\').output.firstRow.translator'
+ type: 'Expression'
+ }
+ }
+ inputs: [
+ {
+ referenceName: dataFactory::dataset_msexports_gzip.name
+ type: 'DatasetReference'
+ parameters: {
+ blobPath: {
+ value: '@pipeline().parameters.blobPath'
+ type: 'Expression'
+ }
+ }
+ }
+ ]
+ outputs: [
+ {
+ referenceName: dataFactory::dataset_ingestion.name
+ type: 'DatasetReference'
+ parameters: {
+ blobPath: {
+ value: '@variables(\'destinationPath\')'
+ type: 'Expression'
+ }
+ }
+ }
+ ]
+ }
+ ]
+ }
+ { // Parquet
+ value: 'parquet'
+ activities: [
+ { // Move Parquet File
+ name: 'Move Parquet File'
+ type: 'Copy'
+ dependsOn: []
+ policy: {
+ timeout: '0.00:05:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ source: {
+ type: 'ParquetSource'
+ additionalColumns: {
+ value: '@variables(\'additionalColumns\')'
+ type: 'Expression'
+ }
+ storeSettings: {
+ type: 'AzureBlobFSReadSettings'
+ recursive: true
+ enablePartitionDiscovery: false
+ }
+ formatSettings: {
+ type: 'ParquetReadSettings'
+ }
+ }
+ sink: {
+ type: 'ParquetSink'
+ storeSettings: {
+ type: 'AzureBlobFSWriteSettings'
+ }
+ formatSettings: {
+ type: 'ParquetWriteSettings'
+ fileExtension: '.parquet'
+ }
+ }
+ enableStaging: false
+ parallelCopies: 1
+ validateDataConsistency: false
+ }
+ inputs: [
+ {
+ referenceName: dataFactory::dataset_msexports_parquet.name
+ type: 'DatasetReference'
+ parameters: {
+ blobPath: {
+ value: '@pipeline().parameters.blobPath'
+ type: 'Expression'
+ }
+ }
+ }
+ ]
+ outputs: [
+ {
+ referenceName: dataFactory::dataset_ingestion.name
+ type: 'DatasetReference'
+ parameters: {
+ blobPath: {
+ value: '@variables(\'destinationPath\')'
+ type: 'Expression'
+ }
+ }
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ defaultActivities: [
+ { // Error: UnsupportedFileType
+ name: 'Unsupported File Type'
+ type: 'Fail'
+ dependsOn: []
+ userProperties: []
+ typeProperties: {
+ message: {
+ value: '@concat(\'Unable to ingest the specified export file because the file type is not supported. File: \', pipeline().parameters.blobPath)'
+ type: 'Expression'
+ }
+ errorCode: 'UnsupportedExportFileType'
+ }
+ }
+ ]
+ }
+ }
+ { // Read Hub Config
+ name: 'Read Hub Config'
+ description: 'Read the hub config to determine if the export should be retained.'
+ type: 'Lookup'
+ dependsOn: []
+ policy: {
+ timeout: '0.12:00:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ source: {
+ type: 'JsonSource'
+ storeSettings: {
+ type: 'AzureBlobFSReadSettings'
+ recursive: false
+ enablePartitionDiscovery: false
+ }
+ formatSettings: {
+ type: 'JsonReadSettings'
+ }
+ }
+ dataset: {
+ referenceName: dataFactory::dataset_config.name
+ type: 'DatasetReference'
+ parameters: {
+ fileName: 'settings.json'
+ folderPath: CONFIG
+ }
+ }
+ }
+ }
+ { // If Not Retaining Exports
+ name: 'If Not Retaining Exports'
+ description: 'If the msexports retention period <= 0, delete the source file. The main reason to keep the source file is to allow for troubleshooting and reprocessing in the future.'
+ type: 'IfCondition'
+ dependsOn: [
+ {
+ activity: 'Convert to Parquet'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ {
+ activity: 'Read Hub Config'
+ dependencyConditions: [
+ 'Completed'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ expression: {
+ value: '@lessOrEquals(coalesce(activity(\'Read Hub Config\').output.firstRow.retention.msexports.days, 0), 0)'
+ type: 'Expression'
+ }
+ ifTrueActivities: [
+ { // Delete Source File
+ name: 'Delete Source File'
+ description: 'Delete the exported data file to keep storage costs down. This file is not referenced by any reporting systems.'
+ type: 'Delete'
+ dependsOn: []
+ policy: {
+ timeout: '0.12:00:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ dataset: {
+ referenceName: dataFactory::dataset_msexports_parquet.name
+ type: 'DatasetReference'
+ parameters: {
+ blobPath: {
+ value: '@pipeline().parameters.blobPath'
+ type: 'Expression'
+ }
+ }
+ }
+ enableLogging: false
+ storeSettings: {
+ type: 'AzureBlobFSReadSettings'
+ recursive: true
+ enablePartitionDiscovery: false
+ }
+ }
+ }
+ ]
+ }
+ }
+ ]
+ parameters: {
+ blobPath: {
+ type: 'String'
+ }
+ destinationFile: {
+ type: 'string'
+ }
+ destinationFolder: {
+ type: 'string'
+ }
+ ingestionId: {
+ type: 'string'
+ }
+ schemaFile: {
+ type: 'string'
+ }
+ exportDatasetType: {
+ type: 'string'
+ }
+ exportDatasetVersion: {
+ type: 'string'
+ }
+ }
+ variables: {
+ additionalColumns: {
+ type: 'Array'
+ }
+ destinationPath: {
+ type: 'String'
+ }
+ }
+ }
+ }
+}
+
+// msexports_ManifestAdded trigger -> msexports_ExecuteETL pipeline
+module trigger_ExportManifestAdded '../../fx/hub-eventTrigger.bicep' = {
+ name: 'Microsoft.CostManagement.Exports_ADF.ExportManifestTrigger'
+ params: {
+ dataFactoryName: dataFactory.name
+ triggerName: '${MSEXPORTS}_ManifestAdded'
+
+ // TODO: Replace pipeline with event: 'Microsoft.CostManagement.Exports.ManifestAdded'
+ pipelineName: dataFactory::pipeline_ExecuteExportsETL.name
+ pipelineParameters: {
+ folderPath: '@triggerBody().folderPath'
+ fileName: '@triggerBody().fileName'
+ }
+
+ storageAccountName: app.storage
+ storageContainer: MSEXPORTS
+ storagePathEndsWith: 'manifest.json'
+ }
+}
+
+
+//==============================================================================
+// Outputs
+//==============================================================================
+
+@description('Properties of the hub app.')
+output app HubAppProperties = app
+
+@description('Name of the container used for Cost Management exports.')
+output exportContainer string = exportContainer.outputs.containerName
+
+@description('Number of schema files uploaded.')
+output schemaFilesUploaded int = schemaFiles.outputs.filesUploaded
diff --git a/src/templates/finops-hub/schemas/actualcost_c360-2025-04.json b/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/actualcost_c360-2025-04.json
similarity index 100%
rename from src/templates/finops-hub/schemas/actualcost_c360-2025-04.json
rename to src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/actualcost_c360-2025-04.json
diff --git a/src/templates/finops-hub/schemas/amortizedcost_c360-2025-04.json b/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/amortizedcost_c360-2025-04.json
similarity index 100%
rename from src/templates/finops-hub/schemas/amortizedcost_c360-2025-04.json
rename to src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/amortizedcost_c360-2025-04.json
diff --git a/src/templates/finops-hub/schemas/focuscost_1.0-preview(v1).json b/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/focuscost_1.0-preview(v1).json
similarity index 100%
rename from src/templates/finops-hub/schemas/focuscost_1.0-preview(v1).json
rename to src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/focuscost_1.0-preview(v1).json
diff --git a/src/templates/finops-hub/schemas/focuscost_1.0.json b/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/focuscost_1.0.json
similarity index 100%
rename from src/templates/finops-hub/schemas/focuscost_1.0.json
rename to src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/focuscost_1.0.json
diff --git a/src/templates/finops-hub/schemas/focuscost_1.0r2.json b/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/focuscost_1.0r2.json
similarity index 100%
rename from src/templates/finops-hub/schemas/focuscost_1.0r2.json
rename to src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/focuscost_1.0r2.json
diff --git a/src/templates/finops-hub/schemas/focuscost_1.2-preview.json b/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/focuscost_1.2-preview.json
similarity index 100%
rename from src/templates/finops-hub/schemas/focuscost_1.2-preview.json
rename to src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/focuscost_1.2-preview.json
diff --git a/src/templates/finops-hub/schemas/focuscost_1.2.json b/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/focuscost_1.2.json
similarity index 100%
rename from src/templates/finops-hub/schemas/focuscost_1.2.json
rename to src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/focuscost_1.2.json
diff --git a/src/templates/finops-hub/schemas/pricesheet_2023-05-01_ea.json b/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/pricesheet_2023-05-01_ea.json
similarity index 100%
rename from src/templates/finops-hub/schemas/pricesheet_2023-05-01_ea.json
rename to src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/pricesheet_2023-05-01_ea.json
diff --git a/src/templates/finops-hub/schemas/pricesheet_2023-05-01_mca.json b/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/pricesheet_2023-05-01_mca.json
similarity index 100%
rename from src/templates/finops-hub/schemas/pricesheet_2023-05-01_mca.json
rename to src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/pricesheet_2023-05-01_mca.json
diff --git a/src/templates/finops-hub/schemas/reservationdetails_2023-03-01.json b/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/reservationdetails_2023-03-01.json
similarity index 100%
rename from src/templates/finops-hub/schemas/reservationdetails_2023-03-01.json
rename to src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/reservationdetails_2023-03-01.json
diff --git a/src/templates/finops-hub/schemas/reservationrecommendations_2023-05-01_ea.json b/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/reservationrecommendations_2023-05-01_ea.json
similarity index 100%
rename from src/templates/finops-hub/schemas/reservationrecommendations_2023-05-01_ea.json
rename to src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/reservationrecommendations_2023-05-01_ea.json
diff --git a/src/templates/finops-hub/schemas/reservationrecommendations_2023-05-01_mca.json b/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/reservationrecommendations_2023-05-01_mca.json
similarity index 100%
rename from src/templates/finops-hub/schemas/reservationrecommendations_2023-05-01_mca.json
rename to src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/reservationrecommendations_2023-05-01_mca.json
diff --git a/src/templates/finops-hub/schemas/reservationtransactions_2023-05-01_ea.json b/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/reservationtransactions_2023-05-01_ea.json
similarity index 100%
rename from src/templates/finops-hub/schemas/reservationtransactions_2023-05-01_ea.json
rename to src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/reservationtransactions_2023-05-01_ea.json
diff --git a/src/templates/finops-hub/schemas/reservationtransactions_2023-05-01_mca.json b/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/reservationtransactions_2023-05-01_mca.json
similarity index 100%
rename from src/templates/finops-hub/schemas/reservationtransactions_2023-05-01_mca.json
rename to src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/reservationtransactions_2023-05-01_mca.json
diff --git a/src/templates/finops-hub/modules/Microsoft.CostManagement/ManagedExports/app.bicep b/src/templates/finops-hub/modules/Microsoft.CostManagement/ManagedExports/app.bicep
new file mode 100644
index 000000000..848fc8aac
--- /dev/null
+++ b/src/templates/finops-hub/modules/Microsoft.CostManagement/ManagedExports/app.bicep
@@ -0,0 +1,1620 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { finOpsToolkitVersion, HubAppProperties } from '../../fx/hub-types.bicep'
+
+
+//==============================================================================
+// Parameters
+//==============================================================================
+
+@description('Required. FinOps hub app getting deployed.')
+param app HubAppProperties
+
+
+//==============================================================================
+// Variables
+//==============================================================================
+
+var CONFIG = 'config'
+var MSEXPORTS = 'msexports'
+
+var exportsApiVersion = '2023-07-01-preview'
+var exportDataVersions = {
+ focuscost: '1.2-preview'
+ pricesheet: '2023-03-01'
+ reservationdetails: '2023-03-01'
+ reservationrecommendations: '2023-05-01'
+ reservationtransactions: '2023-05-01'
+}
+
+// cSpell:ignore timeframe
+// Function to generate the body for a Cost Management export
+func getExportBody(exportContainerName string, datasetType string, schemaVersion string, isMonthly bool, exportFormat string, compressionMode string, partitionData string, dataOverwriteBehavior string) string => '{ "properties": { "definition": { "dataSet": { "configuration": { "dataVersion": "${schemaVersion}", "filters": [] }, "granularity": "Daily" }, "timeframe": "${isMonthly ? 'TheLastMonth': 'MonthToDate' }", "type": "${datasetType}" }, "deliveryInfo": { "destination": { "container": "${exportContainerName}", "rootFolderPath": "@{if(startswith(item().scope, \'/\'), substring(item().scope, 1, sub(length(item().scope), 1)) ,item().scope)}", "type": "AzureBlob", "resourceId": "@{variables(\'storageAccountId\')}" } }, "schedule": { "recurrence": "${ isMonthly ? 'Monthly' : 'Daily'}", "recurrencePeriod": { "from": "2024-01-01T00:00:00.000Z", "to": "2050-02-01T00:00:00.000Z" }, "status": "Inactive" }, "format": "${exportFormat}", "partitionData": "${partitionData}", "dataOverwriteBehavior": "${dataOverwriteBehavior}", "compressionMode": "${compressionMode}" }, "id": "@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{variables(\'exportName\')}", "name": "@{variables(\'exportName\')}", "type": "Microsoft.CostManagement/reports", "identity": { "type": "systemAssigned" }, "location": "global" }'
+
+func getExportBodyV2(exportContainerName string, datasetType string, isMonthly bool, exportFormat string, compressionMode string, partitionData string, dataOverwriteBehavior string, recommendationScope string, recommendationLookbackPeriod string, resourceType string) string => /*
+ */ toLower(datasetType) == 'focuscost' ? /*
+ */ '{ "properties": { "definition": { "dataSet": { "configuration": { "dataVersion": "${exportDataVersions[toLower(datasetType)]}", "filters": [] }, "granularity": "Daily" }, "timeframe": "${isMonthly ? 'TheLastMonth': 'MonthToDate' }", "type": "${datasetType}" }, "deliveryInfo": { "destination": { "container": "${exportContainerName}", "rootFolderPath": "@{if(startswith(item().scope, \'/\'), substring(item().scope, 1, sub(length(item().scope), 1)) ,item().scope)}", "type": "AzureBlob", "resourceId": "@{variables(\'storageAccountId\')}" } }, "schedule": { "recurrence": "${ isMonthly ? 'Monthly' : 'Daily'}", "recurrencePeriod": { "from": "2024-01-01T00:00:00.000Z", "to": "2050-02-01T00:00:00.000Z" }, "status": "Inactive" }, "format": "${exportFormat}", "partitionData": "${partitionData}", "dataOverwriteBehavior": "${dataOverwriteBehavior}", "compressionMode": "${compressionMode}" }, "id": "@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{toLower(concat(variables(\'finOpsHub\'), \'-${ isMonthly ? 'monthly' : 'daily'}-costdetails\'))}", "name": "@{toLower(concat(variables(\'finOpsHub\'), \'-${ isMonthly ? 'monthly' : 'daily'}-costdetails\'))}", "type": "Microsoft.CostManagement/reports", "identity": { "type": "systemAssigned" }, "location": "global" }' /*
+ */ : toLower(datasetType) == 'reservationdetails' ? /*
+ */ '{ "properties": { "definition": { "dataSet": { "configuration": { "dataVersion": "${exportDataVersions[toLower(datasetType)]}", "filters": [] }, "granularity": "Daily" }, "timeframe": "${isMonthly ? 'TheLastMonth': 'MonthToDate' }", "type": "${datasetType}" }, "deliveryInfo": { "destination": { "container": "${exportContainerName}", "rootFolderPath": "@{if(startswith(item().scope, \'/\'), substring(item().scope, 1, sub(length(item().scope), 1)) ,item().scope)}", "type": "AzureBlob", "resourceId": "@{variables(\'storageAccountId\')}" } }, "schedule": { "recurrence": "${ isMonthly ? 'Monthly' : 'Daily'}", "recurrencePeriod": { "from": "2024-01-01T00:00:00.000Z", "to": "2050-02-01T00:00:00.000Z" }, "status": "Inactive" }, "format": "${exportFormat}", "partitionData": "${partitionData}", "dataOverwriteBehavior": "${dataOverwriteBehavior}", "compressionMode": "${compressionMode}" }, "id": "@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{toLower(concat(variables(\'finOpsHub\'), \'-${ isMonthly ? 'monthly' : 'daily'}-${toLower(datasetType)}\'))}", "name": "@{toLower(concat(variables(\'finOpsHub\'), \'-${ isMonthly ? 'monthly' : 'daily'}-${toLower(datasetType)}\'))}", "type": "Microsoft.CostManagement/reports", "identity": { "type": "systemAssigned" }, "location": "global" }' /*
+ */ : (toLower(datasetType) == 'pricesheet') || (toLower(datasetType) == 'reservationtransactions') ? /*
+ */ '{ "properties": { "definition": { "dataSet": { "configuration": { "dataVersion": "${exportDataVersions[toLower(datasetType)]}", "filters": [] }}, "timeframe": "${isMonthly ? 'TheCurrentMonth': 'MonthToDate' }", "type": "${datasetType}" }, "deliveryInfo": { "destination": { "container": "${exportContainerName}", "rootFolderPath": "@{if(startswith(item().scope, \'/\'), substring(item().scope, 1, sub(length(item().scope), 1)) ,item().scope)}", "type": "AzureBlob", "resourceId": "@{variables(\'storageAccountId\')}" } }, "schedule": { "recurrence": "${ isMonthly ? 'Monthly' : 'Daily'}", "recurrencePeriod": { "from": "2024-01-01T00:00:00.000Z", "to": "2050-02-01T00:00:00.000Z" }, "status": "Inactive" }, "format": "${exportFormat}", "partitionData": "${partitionData}", "dataOverwriteBehavior": "${dataOverwriteBehavior}", "compressionMode": "${compressionMode}" }, "id": "@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{toLower(concat(variables(\'finOpsHub\'), \'-${ isMonthly ? 'monthly' : 'daily'}-${toLower(datasetType)}\'))}", "name": "@{toLower(concat(variables(\'finOpsHub\'), \'-${ isMonthly ? 'monthly' : 'daily'}-${toLower(datasetType)}\'))}", "type": "Microsoft.CostManagement/reports", "identity": { "type": "systemAssigned" }, "location": "global" }' /*
+ */ : toLower(datasetType) == 'reservationrecommendations' ? /*
+ */ '{ "properties": { "definition": { "dataSet": { "configuration": { "dataVersion": "${exportDataVersions[toLower(datasetType)]}", "filters": [ { "name": "reservationScope", "value": "${recommendationScope}" }, { "name": "resourceType", "value": "${resourceType}" }, { "name": "lookBackPeriod", "value": "${recommendationLookbackPeriod}" }] }}, "timeframe": "${isMonthly ? 'TheLastMonth': 'MonthToDate' }", "type": "${datasetType}" }, "deliveryInfo": { "destination": { "container": "${exportContainerName}", "rootFolderPath": "@{if(startswith(item().scope, \'/\'), substring(item().scope, 1, sub(length(item().scope), 1)) ,item().scope)}", "type": "AzureBlob", "resourceId": "@{variables(\'storageAccountId\')}" } }, "schedule": { "recurrence": "${ isMonthly ? 'Monthly' : 'Daily'}", "recurrencePeriod": { "from": "2024-01-01T00:00:00.000Z", "to": "2050-02-01T00:00:00.000Z" }, "status": "Inactive" }, "format": "${exportFormat}", "partitionData": "${partitionData}", "dataOverwriteBehavior": "${dataOverwriteBehavior}", "compressionMode": "${compressionMode}" }, "id": "@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{toLower(concat(variables(\'finOpsHub\'), \'-${ isMonthly ? 'monthly' : 'daily'}-costdetails\'))}", "name": "@{toLower(concat(variables(\'finOpsHub\'), \'-${ isMonthly ? 'monthly' : 'daily'}-costdetails\'))}", "type": "Microsoft.CostManagement/reports", "identity": { "type": "systemAssigned" }, "location": "global" }' /*
+ */ : 'undefined'
+
+//==============================================================================
+// Resources
+//==============================================================================
+
+// Register app
+module appRegistration '../../fx/hub-app.bicep' = {
+ name: 'Microsoft.CostManagement.ManagedExports_Register'
+ params: {
+ app: app
+ version: finOpsToolkitVersion
+ features: [
+ 'DataFactory'
+ ]
+ }
+}
+
+resource storageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' existing = {
+ name: app.storage
+}
+
+resource dataFactory 'Microsoft.DataFactory/factories@2018-06-01' existing = {
+ name: app.dataFactory
+
+ resource dataset_config 'datasets' existing = {
+ name: CONFIG
+ }
+
+ resource trigger_DailySchedule 'triggers' = {
+ name: '${CONFIG}_DailySchedule'
+ properties: {
+ pipelines: [
+ {
+ pipelineReference: {
+ referenceName: dataFactory::pipeline_StartExportProcess.name
+ type: 'PipelineReference'
+ }
+ parameters: {
+ Recurrence: 'Daily'
+ }
+ }
+ ]
+ type: 'ScheduleTrigger'
+ typeProperties: {
+ recurrence: {
+ frequency: 'Hour'
+ interval: 24
+ startTime: '2023-01-01T01:01:00'
+ timeZone: timeZones.outputs.Timezone
+ }
+ }
+ }
+ }
+
+ resource trigger_MonthlySchedule 'triggers' = {
+ name: '${CONFIG}_MonthlySchedule'
+ properties: {
+ pipelines: [
+ {
+ pipelineReference: {
+ referenceName: dataFactory::pipeline_StartExportProcess.name
+ type: 'PipelineReference'
+ }
+ parameters: {
+ Recurrence: 'Monthly'
+ }
+ }
+ ]
+ type: 'ScheduleTrigger'
+ typeProperties: {
+ recurrence: {
+ frequency: 'Month'
+ interval: 1
+ startTime: '2023-01-05T01:11:00'
+ timeZone: timeZones.outputs.Timezone
+ schedule: {
+ monthDays: [
+ 2
+ 5
+ 19
+ ]
+ }
+ }
+ }
+ }
+ }
+
+ //----------------------------------------------------------------------------
+ // config_StartBackfillProcess pipeline
+ //----------------------------------------------------------------------------
+ resource pipeline_StartBackfillProcess 'pipelines' = {
+ name: '${CONFIG}_StartBackfillProcess'
+ properties: {
+ activities: [
+ { // Get Config
+ name: 'Get Config'
+ type: 'Lookup'
+ dependsOn: []
+ policy: {
+ timeout: '0.00:05:00'
+ retry: 2
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ source: {
+ type: 'JsonSource'
+ storeSettings: {
+ type: 'AzureBlobFSReadSettings'
+ recursive: true
+ enablePartitionDiscovery: false
+ }
+ formatSettings: {
+ type: 'JsonReadSettings'
+ }
+ }
+ dataset: {
+ referenceName: dataset_config.name
+ type: 'DatasetReference'
+ parameters: {
+ fileName: {
+ value: '@variables(\'fileName\')'
+ type: 'Expression'
+ }
+ folderPath: {
+ value: '@variables(\'folderPath\')'
+ type: 'Expression'
+ }
+ }
+ }
+ }
+ }
+ { // Set backfill end date
+ name: 'Set backfill end date'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Get Config'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ variableName: 'endDate'
+ value: {
+ value: '@addDays(startOfMonth(utcNow()), -1)'
+ type: 'Expression'
+ }
+ }
+ }
+ { // Set backfill start date
+ name: 'Set backfill start date'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Get Config'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ variableName: 'startDate'
+ value: {
+ value: '@subtractFromTime(startOfMonth(utcNow()), activity(\'Get Config\').output.firstRow.retention.ingestion.months, \'Month\')'
+ type: 'Expression'
+ }
+ }
+ }
+ { // Set export start date
+ name: 'Set export start date'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Set backfill start date'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ variableName: 'thisMonth'
+ value: {
+ value: '@startOfMonth(variables(\'endDate\'))'
+ type: 'Expression'
+ }
+ }
+ }
+ { // Set export end date
+ name: 'Set export end date'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Set export start date'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ variableName: 'nextMonth'
+ value: {
+ value: '@startOfMonth(subtractFromTime(variables(\'thisMonth\'), 1, \'Month\'))'
+ type: 'Expression'
+ }
+ }
+ }
+ { // Every Month
+ name: 'Every Month'
+ type: 'Until'
+ dependsOn: [
+ {
+ activity: 'Set export end date'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ {
+ activity: 'Set backfill end date'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ expression: {
+ value: '@less(variables(\'thisMonth\'), variables(\'startDate\'))'
+ type: 'Expression'
+ }
+ activities: [
+ {
+ name: 'Update export start date'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Backfill data'
+ dependencyConditions: [
+ 'Completed'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ variableName: 'thisMonth'
+ value: {
+ value: '@variables(\'nextMonth\')'
+ type: 'Expression'
+ }
+ }
+ }
+ {
+ name: 'Update export end date'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Update export start date'
+ dependencyConditions: [
+ 'Completed'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ variableName: 'nextMonth'
+ value: {
+ value: '@subtractFromTime(variables(\'thisMonth\'), 1, \'Month\')'
+ type: 'Expression'
+ }
+ }
+ }
+ {
+ name: 'Backfill data'
+ type: 'ExecutePipeline'
+ dependsOn: []
+ userProperties: []
+ typeProperties: {
+ pipeline: {
+ referenceName: pipeline_RunBackfillJob.name
+ type: 'PipelineReference'
+ }
+ waitOnCompletion: true
+ parameters: {
+ StartDate: {
+ value: '@variables(\'thisMonth\')'
+ type: 'Expression'
+ }
+ EndDate: {
+ value: '@addDays(addToTime(variables(\'thisMonth\'), 1, \'Month\'), -1)'
+ type: 'Expression'
+ }
+ }
+ }
+ }
+ ]
+ timeout: '0.02:00:00'
+ }
+ }
+ ]
+ concurrency: 1
+ variables: {
+ exportName: {
+ type: 'String'
+ }
+ storageAccountId: {
+ type: 'String'
+ defaultValue: storageAccount.id
+ }
+ finOpsHub: {
+ type: 'String'
+ defaultValue: app.hub.name
+ }
+ resourceManagementUri: {
+ type: 'String'
+ defaultValue: environment().resourceManager
+ }
+ fileName: {
+ type: 'String'
+ defaultValue: 'settings.json'
+ }
+ folderPath: {
+ type: 'String'
+ defaultValue: CONFIG
+ }
+ endDate: {
+ type: 'String'
+ }
+ startDate: {
+ type: 'String'
+ }
+ thisMonth: {
+ type: 'String'
+ }
+ nextMonth: {
+ type: 'String'
+ }
+ }
+ }
+ }
+
+ //----------------------------------------------------------------------------
+ // config_RunBackfillJob pipeline
+ // Triggered by config_StartBackfillProcess pipeline
+ //----------------------------------------------------------------------------
+ resource pipeline_RunBackfillJob 'pipelines' = {
+ name: '${CONFIG}_RunBackfillJob'
+ properties: {
+ activities: [
+ { // Get Config
+ name: 'Get Config'
+ type: 'Lookup'
+ dependsOn: []
+ policy: {
+ timeout: '0.00:05:00'
+ retry: 2
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ source: {
+ type: 'JsonSource'
+ storeSettings: {
+ type: 'AzureBlobFSReadSettings'
+ recursive: true
+ enablePartitionDiscovery: false
+ }
+ formatSettings: {
+ type: 'JsonReadSettings'
+ }
+ }
+ dataset: {
+ referenceName: dataset_config.name
+ type: 'DatasetReference'
+ parameters: {
+ fileName: {
+ value: '@variables(\'fileName\')'
+ type: 'Expression'
+ }
+ folderPath: {
+ value: '@variables(\'folderPath\')'
+ type: 'Expression'
+ }
+ }
+ }
+ }
+ }
+ { // Set Scopes
+ name: 'Set Scopes'
+ description: 'Save scopes to test if it is an array'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Get Config'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'scopesArray'
+ value: {
+ value: '@activity(\'Get Config\').output.firstRow.scopes'
+ type: 'Expression'
+ }
+ }
+ }
+ { // Set Scopes as Array
+ name: 'Set Scopes as Array'
+ description: 'Wraps a single scope object into an array to work around the PowerShell bug where single-item arrays are sometimes written as a single object instead of an array.'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Set Scopes'
+ dependencyConditions: [
+ 'Failed'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'scopesArray'
+ value: {
+ value: '@createArray(activity(\'Get Config\').output.firstRow.scopes)'
+ type: 'Expression'
+ }
+ }
+ }
+ { // Filter Invalid Scopes
+ name: 'Filter Invalid Scopes'
+ description: 'Remove any invalid scopes to avoid errors.'
+ type: 'Filter'
+ dependsOn: [
+ {
+ activity: 'Set Scopes'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ {
+ activity: 'Set Scopes as Array'
+ dependencyConditions: [
+ 'Skipped'
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ items: {
+ value: '@variables(\'scopesArray\')'
+ type: 'Expression'
+ }
+ condition: {
+ value: '@and(not(empty(item().scope)), not(equals(item().scope, \'/\')))'
+ type: 'Expression'
+ }
+ }
+ }
+ { // ForEach Export Scope
+ name: 'ForEach Export Scope'
+ type: 'ForEach'
+ dependsOn: [
+ {
+ activity: 'Filter Invalid Scopes'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ items: {
+ value: '@activity(\'Filter Invalid Scopes\').output.Value'
+ type: 'Expression'
+ }
+ isSequential: true
+ activities: [
+ {
+ name: 'Set backfill export name'
+ type: 'SetVariable'
+ dependsOn: []
+ userProperties: []
+ typeProperties: {
+ variableName: 'exportName'
+ value: {
+ // cSpell:ignore costdetails
+ value: '@toLower(concat(variables(\'finOpsHub\'), \'-monthly-costdetails\'))'
+ type: 'Expression'
+ }
+ }
+ }
+ {
+ name: 'Trigger backfill export'
+ type: 'WebActivity'
+ dependsOn: [
+ {
+ activity: 'Set backfill export name'
+ dependencyConditions: [
+ 'Completed'
+ ]
+ }
+ ]
+ policy: {
+ timeout: '0.00:05:00'
+ retry: 1
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ url: {
+ value: '@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{variables(\'exportName\')}/run?api-version=${exportsApiVersion}'
+ type: 'Expression'
+ }
+ method: 'POST'
+ headers: {
+ 'x-ms-command-name': 'FinOpsToolkit.Hubs.config_RunBackfill@${finOpsToolkitVersion}'
+ 'Content-Type': 'application/json'
+ ClientType: 'FinOpsToolkit.Hubs@${finOpsToolkitVersion}'
+ }
+ body: '{"timePeriod" : { "from" : "@{pipeline().parameters.StartDate}", "to" : "@{pipeline().parameters.EndDate}" }}'
+ authentication: {
+ type: 'MSI'
+ resource: {
+ value: '@variables(\'resourceManagementUri\')'
+ type: 'Expression'
+ }
+ }
+ }
+ }
+ ]
+ }
+ }
+ ]
+ concurrency: 1
+ parameters: {
+ StartDate: {
+ type: 'string'
+ }
+ EndDate: {
+ type: 'string'
+ }
+ }
+ variables: {
+ exportName: {
+ type: 'String'
+ }
+ storageAccountId: {
+ type: 'String'
+ defaultValue: storageAccount.id
+ }
+ finOpsHub: {
+ type: 'String'
+ defaultValue: app.hub.name
+ }
+ resourceManagementUri: {
+ type: 'String'
+ defaultValue: environment().resourceManager
+ }
+ fileName: {
+ type: 'String'
+ defaultValue: 'settings.json'
+ }
+ folderPath: {
+ type: 'String'
+ defaultValue: CONFIG
+ }
+ scopesArray: {
+ type: 'Array'
+ }
+ }
+ }
+ }
+
+ //----------------------------------------------------------------------------
+ // config_StartExportProcess pipeline
+ // Triggered by config_DailySchedule/MonthlySchedule triggers
+ //----------------------------------------------------------------------------
+ resource pipeline_StartExportProcess 'pipelines' = {
+ name: '${CONFIG}_StartExportProcess'
+ properties: {
+ activities: [
+ { // Get Config
+ name: 'Get Config'
+ type: 'Lookup'
+ dependsOn: []
+ policy: {
+ timeout: '0.00:05:00'
+ retry: 2
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ source: {
+ type: 'JsonSource'
+ storeSettings: {
+ type: 'AzureBlobFSReadSettings'
+ recursive: true
+ enablePartitionDiscovery: false
+ }
+ formatSettings: {
+ type: 'JsonReadSettings'
+ }
+ }
+ dataset: {
+ referenceName: dataset_config.name
+ type: 'DatasetReference'
+ parameters: {
+ fileName: {
+ value: '@variables(\'fileName\')'
+ type: 'Expression'
+ }
+ folderPath: {
+ value: '@variables(\'folderPath\')'
+ type: 'Expression'
+ }
+ }
+ }
+ }
+ }
+ { // Set Scopes
+ name: 'Set Scopes'
+ description: 'Save scopes to test if it is an array'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Get Config'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'scopesArray'
+ value: {
+ value: '@activity(\'Get Config\').output.firstRow.scopes'
+ type: 'Expression'
+ }
+ }
+ }
+ { // Set Scopes as Array
+ name: 'Set Scopes as Array'
+ description: 'Wraps a single scope object into an array to work around the PowerShell bug where single-item arrays are sometimes written as a single object instead of an array.'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Set Scopes'
+ dependencyConditions: [
+ 'Failed'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'scopesArray'
+ value: {
+ value: '@createArray(activity(\'Get Config\').output.firstRow.scopes)'
+ type: 'Expression'
+ }
+ }
+ }
+ { // Filter Invalid Scopes
+ name: 'Filter Invalid Scopes'
+ description: 'Remove any invalid scopes to avoid errors.'
+ type: 'Filter'
+ dependsOn: [
+ {
+ activity: 'Set Scopes'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ {
+ activity: 'Set Scopes as Array'
+ dependencyConditions: [
+ 'Succeeded'
+ 'Skipped'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ items: {
+ value: '@variables(\'scopesArray\')'
+ type: 'Expression'
+ }
+ condition: {
+ value: '@and(not(empty(item().scope)), not(equals(item().scope, \'/\')))'
+ type: 'Expression'
+ }
+ }
+ }
+ { // ForEach Export Scope
+ name: 'ForEach Export Scope'
+ type: 'ForEach'
+ dependsOn: [
+ {
+ activity: 'Filter Invalid Scopes'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ items: {
+ value: '@activity(\'Filter Invalid Scopes\').output.Value'
+ type: 'Expression'
+ }
+ isSequential: true
+ activities: [
+ {
+ name: 'Get exports for scope'
+ type: 'WebActivity'
+ dependsOn: []
+ policy: {
+ timeout: '0.00:05:00'
+ retry: 2
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ url: {
+ value: '@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports?api-version=${exportsApiVersion}'
+ type: 'Expression'
+ }
+ method: 'GET'
+ authentication: {
+ type: 'MSI'
+ resource: {
+ value: '@variables(\'resourceManagementUri\')'
+ type: 'Expression'
+ }
+ }
+ }
+ }
+ {
+ name: 'Run exports for scope'
+ type: 'ExecutePipeline'
+ dependsOn: [
+ {
+ activity: 'Get exports for scope'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ pipeline: {
+ referenceName: pipeline_RunExportJobs.name
+ type: 'PipelineReference'
+ }
+ waitOnCompletion: true
+ parameters: {
+ ExportScopes: {
+ value: '@activity(\'Get exports for scope\').output.value'
+ type: 'Expression'
+ }
+ Recurrence: {
+ value: '@pipeline().parameters.Recurrence'
+ type: 'Expression'
+ }
+ }
+ }
+ }
+ ]
+ }
+ }
+ ]
+ concurrency: 1
+ parameters: {
+ Recurrence: {
+ type: 'string'
+ defaultValue: 'Daily'
+ }
+ }
+ variables: {
+ fileName: {
+ type: 'String'
+ defaultValue: 'settings.json'
+ }
+ folderPath: {
+ type: 'String'
+ defaultValue: CONFIG
+ }
+ finOpsHub: {
+ type: 'String'
+ defaultValue: app.hub.name
+ }
+ resourceManagementUri: {
+ type: 'String'
+ defaultValue: environment().resourceManager
+ }
+ scopesArray: {
+ type: 'Array'
+ }
+ }
+ }
+ }
+
+ //----------------------------------------------------------------------------
+ // config_RunExportJobs pipeline
+ // Triggered by pipeline_StartExportProcess pipeline
+ //----------------------------------------------------------------------------
+ resource pipeline_RunExportJobs 'pipelines' = {
+ name: '${CONFIG}_RunExportJobs'
+ dependsOn: [
+ dataset_config
+ ]
+ properties: {
+ activities: [
+ {
+ name: 'ForEach export scope'
+ type: 'ForEach'
+ dependsOn: []
+ userProperties: []
+ typeProperties: {
+ items: {
+ value: '@pipeline().parameters.exportScopes'
+ type: 'Expression'
+ }
+ isSequential: true
+ activities: [
+ {
+ name: 'If scheduled'
+ type: 'IfCondition'
+ dependsOn: []
+ userProperties: []
+ typeProperties: {
+ expression: {
+ value: '@and( startswith(toLower(item().name), toLower(variables(\'hubName\'))), and(contains(string(item().properties.schedule), \'recurrence\'), equals(toLower(item().properties.schedule.recurrence), toLower(pipeline().parameters.Recurrence))))'
+ type: 'Expression'
+ }
+ ifTrueActivities: [
+ {
+ name: 'Trigger export'
+ type: 'WebActivity'
+ dependsOn: []
+ policy: {
+ timeout: '0.00:05:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ method: 'POST'
+ url: {
+ value: '@{replace(toLower(concat(variables(\'resourceManagementUri\'),item().id)), \'com//\', \'com/\')}/run?api-version=${exportsApiVersion}'
+ type: 'Expression'
+ }
+ headers: {
+ 'x-ms-command-name': 'FinOpsToolkit.Hubs.config_RunExportJobs@${finOpsToolkitVersion}'
+ ClientType: 'FinOpsToolkit.Hubs@${finOpsToolkitVersion}'
+ }
+ body: ' '
+ authentication: {
+ type: 'MSI'
+ resource: {
+ value: '@variables(\'resourceManagementUri\')'
+ type: 'Expression'
+ }
+ }
+ }
+ }
+ ]
+ }
+ }
+ ]
+ }
+ }
+ ]
+ concurrency: 1
+ parameters: {
+ ExportScopes: {
+ type: 'array'
+ }
+ Recurrence: {
+ type: 'string'
+ defaultValue: 'Daily'
+ }
+ }
+ variables: {
+ resourceManagementUri: {
+ type: 'String'
+ defaultValue: environment().resourceManager
+ }
+ hubName: {
+ type: 'String'
+ defaultValue: app.hub.name
+ }
+ }
+ }
+ }
+
+ //----------------------------------------------------------------------------
+ // config_ConfigureExports pipeline
+ // Triggered by config_SettingsUpdated trigger
+ //----------------------------------------------------------------------------
+ resource pipeline_ConfigureExports 'pipelines' = {
+ name: '${CONFIG}_ConfigureExports'
+ properties: {
+ activities: [
+ { // Get Config
+ name: 'Get Config'
+ type: 'Lookup'
+ dependsOn: []
+ policy: {
+ timeout: '0.00:05:00'
+ retry: 2
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ source: {
+ type: 'JsonSource'
+ storeSettings: {
+ type: 'AzureBlobFSReadSettings'
+ recursive: true
+ enablePartitionDiscovery: false
+ }
+ formatSettings: {
+ type: 'JsonReadSettings'
+ }
+ }
+ dataset: {
+ referenceName: dataset_config.name
+ type: 'DatasetReference'
+ parameters: {
+ fileName: {
+ value: '@variables(\'fileName\')'
+ type: 'Expression'
+ }
+ folderPath: {
+ value: '@variables(\'folderPath\')'
+ type: 'Expression'
+ }
+ }
+ }
+ }
+ }
+ { // Save Scopes
+ name: 'Save Scopes'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Get Config'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'scopesArray'
+ value: {
+ value: '@activity(\'Get Config\').output.firstRow.scopes'
+ type: 'Expression'
+ }
+ }
+ }
+ { // Save Scopes as Array
+ name: 'Save Scopes as Array'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Save Scopes'
+ dependencyConditions: [
+ 'Failed'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'scopesArray'
+ value: {
+ value: '@array(activity(\'Get Config\').output.firstRow.scopes)'
+ type: 'Expression'
+ }
+ }
+ }
+ { // Filter Invalid Scopes
+ name: 'Filter Invalid Scopes'
+ type: 'Filter'
+ dependsOn: [
+ {
+ activity: 'Save Scopes'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ {
+ activity: 'Save Scopes as Array'
+ dependencyConditions: [
+ 'Skipped'
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ items: {
+ value: '@variables(\'scopesArray\')'
+ type: 'Expression'
+ }
+ condition: {
+ value: '@and(not(empty(item().scope)), not(equals(item().scope, \'/\')))'
+ type: 'Expression'
+ }
+ }
+ }
+ { // ForEach Export Scope
+ name: 'ForEach Export Scope'
+ type: 'ForEach'
+ dependsOn: [
+ {
+ activity: 'Filter Invalid Scopes'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ items: {
+ value: '@activity(\'Filter Invalid Scopes\').output.value'
+ type: 'Expression'
+ }
+ isSequential: true
+ activities: [
+ {
+ name: 'Set Export Type'
+ type: 'SetVariable'
+ dependsOn: []
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'exportScopeType'
+ value: {
+ value: '@if(contains(toLower(item().scope), \'providers/microsoft.billing/billingaccounts\'), if(contains(toLower(item().scope), \':\'), \'mca\', \'ea\'), if(contains(toLower(item().scope), \'subscriptions/\'), \'subscription\', \'undefined\'))'
+ type: 'Expression'
+ }
+ }
+ }
+ {
+ name: 'Switch Export Type'
+ type: 'Switch'
+ dependsOn: [
+ {
+ activity: 'Set Export Type'
+ dependencyConditions: [ 'Succeeded' ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ on: {
+ value: '@toLower(variables(\'exportScopeType\'))'
+ type: 'Expression'
+ }
+ cases: [
+ { // EA
+ value: 'ea'
+ activities: [
+ { // 'Open month focus export'
+ name: 'Open month focus export'
+ type: 'WebActivity'
+ dependsOn: [
+ ]
+ policy: {
+ timeout: '0.00:05:00'
+ retry: 2
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ url: {
+ value: '@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{toLower(concat(variables(\'finOpsHub\'), \'-daily-costdetails\'))}?api-version=${exportsApiVersion}'
+ type: 'Expression'
+ }
+ method: 'PUT'
+ body: {
+ value: getExportBodyV2(MSEXPORTS, 'FocusCost', false, 'Parquet', 'Snappy', 'true', 'CreateNewReport', '', '', '')
+ type: 'Expression'
+ }
+ headers: {
+ 'x-ms-command-name': 'FinOpsToolkit.Hubs.config_RunExportJobs.CostsDaily@${finOpsToolkitVersion}'
+ ClientType: 'FinOpsToolkit.Hubs@${finOpsToolkitVersion}'
+ }
+ authentication: {
+ type: 'MSI'
+ resource: {
+ value: '@variables(\'resourceManagementUri\')'
+ type: 'Expression'
+ }
+ }
+ }
+ }
+ { // 'Closed month focus export'
+ name: 'Closed month focus export'
+ type: 'WebActivity'
+ dependsOn: [
+ {
+ activity: 'Open month focus export'
+ dependencyConditions: [ 'Succeeded' ]
+ }
+ ]
+ policy: {
+ timeout: '0.00:05:00'
+ retry: 2
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ url: {
+ value: '@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{toLower(concat(variables(\'finOpsHub\'), \'-monthly-costdetails\'))}?api-version=${exportsApiVersion}'
+ type: 'Expression'
+ }
+ method: 'PUT'
+ body: {
+ value: getExportBodyV2(MSEXPORTS, 'FocusCost', true, 'Parquet', 'Snappy', 'true', 'CreateNewReport', '', '', '')
+ type: 'Expression'
+ }
+ headers: {
+ 'x-ms-command-name': 'FinOpsToolkit.Hubs.config_RunExportJobs.CostsMonthly@${finOpsToolkitVersion}'
+ ClientType: 'FinOpsToolkit.Hubs@${finOpsToolkitVersion}'
+ }
+ authentication: {
+ type: 'MSI'
+ resource: {
+ value: '@variables(\'resourceManagementUri\')'
+ type: 'Expression'
+ }
+ }
+ }
+ }
+ { // 'Monthly pricesheet export'
+ name: 'Monthly pricesheet export'
+ type: 'WebActivity'
+ dependsOn: [
+ {
+ activity: 'Closed month focus export'
+ dependencyConditions: [ 'Succeeded' ]
+ }
+ ]
+ policy: {
+ timeout: '0.00:05:00'
+ retry: 2
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ url: {
+ value: '@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{toLower(concat(variables(\'finOpsHub\'), \'-monthly-pricesheet\'))}?api-version=${exportsApiVersion}'
+ type: 'Expression'
+ }
+ method: 'PUT'
+ body: {
+ value: getExportBodyV2(MSEXPORTS, 'Pricesheet', true, 'Parquet', 'Snappy', 'true', 'CreateNewReport', '', '', '')
+ type: 'Expression'
+ }
+ headers: {
+ 'x-ms-command-name': 'FinOpsToolkit.Hubs.config_RunExportJobs.Prices@${finOpsToolkitVersion}'
+ ClientType: 'FinOpsToolkit.Hubs@${finOpsToolkitVersion}'
+ }
+ authentication: {
+ type: 'MSI'
+ resource: {
+ value: '@variables(\'resourceManagementUri\')'
+ type: 'Expression'
+ }
+ }
+ }
+ }
+ {
+ name: 'Trigger EA monthly pricesheet export'
+ type: 'WebActivity'
+ dependsOn: [
+ {
+ activity: 'Monthly pricesheet export'
+ dependencyConditions: [ 'Succeeded' ]
+ }
+ ]
+ policy: {
+ timeout: '0.00:05:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ method: 'POST'
+ url: {
+ value: '@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{toLower(concat(variables(\'finOpsHub\'), \'-monthly-pricesheet\'))}/run?api-version=${exportsApiVersion}'
+ type: 'Expression'
+ }
+ headers: {
+ 'x-ms-command-name': 'FinOpsToolkit.Hubs.config_RunExportJobs.Prices@${finOpsToolkitVersion}'
+ ClientType: 'FinOpsToolkit.Hubs@${finOpsToolkitVersion}'
+ }
+ body: ' '
+ authentication: {
+ type: 'MSI'
+ resource: {
+ value: '@variables(\'resourceManagementUri\')'
+ type: 'Expression'
+ }
+ }
+ }
+ }
+ { // 'Daily reservation details export'
+ name: 'Daily reservation details export'
+ type: 'WebActivity'
+ dependsOn: [
+ {
+ activity: 'Monthly pricesheet export'
+ dependencyConditions: [ 'Succeeded' ]
+ }
+ ]
+ policy: {
+ timeout: '0.00:05:00'
+ retry: 2
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ url: {
+ value: '@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{toLower(concat(variables(\'finOpsHub\'), \'-daily-reservationdetails\'))}?api-version=${exportsApiVersion}'
+ type: 'Expression'
+ }
+ method: 'PUT'
+ body: {
+ value: getExportBodyV2(MSEXPORTS, 'ReservationDetails', false, 'CSV', 'None', 'true', 'CreateNewReport', '', '', '')
+ type: 'Expression'
+ }
+ headers: {
+ 'x-ms-command-name': 'FinOpsToolkit.Hubs.config_RunExportJobs.ReservationDetails@${finOpsToolkitVersion}'
+ ClientType: 'FinOpsToolkit.Hubs@${finOpsToolkitVersion}'
+ }
+ authentication: {
+ type: 'MSI'
+ resource: {
+ value: '@variables(\'resourceManagementUri\')'
+ type: 'Expression'
+ }
+ }
+ }
+ }
+ { // 'Daily reservation transactions export'
+ name: 'Daily reservation transactions export'
+ type: 'WebActivity'
+ dependsOn: [
+ {
+ activity: 'Daily reservation details export'
+ dependencyConditions: [ 'Succeeded' ]
+ }
+ ]
+ policy: {
+ timeout: '0.00:05:00'
+ retry: 2
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ url: {
+ value: '@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{toLower(concat(variables(\'finOpsHub\'), \'-daily-reservationtransactions\'))}?api-version=${exportsApiVersion}'
+ type: 'Expression'
+ }
+ method: 'PUT'
+ body: {
+ value: getExportBodyV2(MSEXPORTS, 'ReservationTransactions', false, 'CSV', 'None', 'true', 'CreateNewReport', '', '', '')
+ type: 'Expression'
+ }
+ headers: {
+ 'x-ms-command-name': 'FinOpsToolkit.Hubs.config_RunExportJobs.ReservationTransactions@${finOpsToolkitVersion}'
+ ClientType: 'FinOpsToolkit.Hubs@${finOpsToolkitVersion}'
+ }
+ authentication: {
+ type: 'MSI'
+ resource: {
+ value: '@variables(\'resourceManagementUri\')'
+ type: 'Expression'
+ }
+ }
+ }
+ }
+ { // 'Daily recommendations shared last30day virtual machines export'
+ name: 'Daily shared 30day virtual machines'
+ type: 'WebActivity'
+ dependsOn: [
+ {
+ activity: 'Daily reservation transactions export'
+ dependencyConditions: [ 'Succeeded' ]
+ }
+ ]
+ policy: {
+ timeout: '0.00:05:00'
+ retry: 2
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ url: {
+ value: '@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{toLower(concat(variables(\'finOpsHub\'), \'-daily-recommendations-shared-last30days-virtualmachines\'))}?api-version=${exportsApiVersion}'
+ type: 'Expression'
+ }
+ method: 'PUT'
+ body: {
+ value: getExportBodyV2(MSEXPORTS, 'ReservationRecommendations', false, 'CSV', 'None', 'true', 'CreateNewReport', 'Shared', 'Last30Days', 'VirtualMachines')
+ type: 'Expression'
+ }
+ headers: {
+ 'x-ms-command-name': 'FinOpsToolkit.Hubs.config_RunExportJobs.ReservationRecommendations.VM.Shared.30d@${finOpsToolkitVersion}'
+ ClientType: 'FinOpsToolkit.Hubs@${finOpsToolkitVersion}'
+ }
+ authentication: {
+ type: 'MSI'
+ resource: {
+ value: '@variables(\'resourceManagementUri\')'
+ type: 'Expression'
+ }
+ }
+ }
+ }
+ ]
+ }
+ { // subscription
+ value: 'subscription'
+ activities: [
+ { // 'Subscription open month focus export'
+ name: 'Subscription open month focus export'
+ type: 'WebActivity'
+ dependsOn: [
+ ]
+ policy: {
+ timeout: '0.00:05:00'
+ retry: 2
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ url: {
+ value: '@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{toLower(concat(variables(\'finOpsHub\'), \'-daily-costdetails\'))}?api-version=${exportsApiVersion}'
+ type: 'Expression'
+ }
+ method: 'PUT'
+ body: {
+ value: getExportBodyV2(MSEXPORTS, 'FocusCost', false, 'Parquet', 'Snappy', 'true', 'CreateNewReport', '', '', '')
+ type: 'Expression'
+ }
+ headers: {
+ 'x-ms-command-name': 'FinOpsToolkit.Hubs.config_RunExportJobs.CostsDaily@${finOpsToolkitVersion}'
+ ClientType: 'FinOpsToolkit.Hubs@${finOpsToolkitVersion}'
+ }
+ authentication: {
+ type: 'MSI'
+ resource: {
+ value: '@variables(\'resourceManagementUri\')'
+ type: 'Expression'
+ }
+ }
+ }
+ }
+ { // 'Subscription closed month focus export'
+ name: 'Subscription closed month focus export'
+ type: 'WebActivity'
+ dependsOn: [
+ {
+ activity: 'Subscription open month focus export'
+ dependencyConditions: [ 'Succeeded' ]
+ }
+ ]
+ policy: {
+ timeout: '0.00:05:00'
+ retry: 2
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ url: {
+ value: '@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{toLower(concat(variables(\'finOpsHub\'), \'-monthly-costdetails\'))}?api-version=${exportsApiVersion}'
+ type: 'Expression'
+ }
+ method: 'PUT'
+ body: {
+ value: getExportBodyV2(MSEXPORTS, 'FocusCost', true, 'Parquet', 'Snappy', 'true', 'CreateNewReport', '', '', '')
+ type: 'Expression'
+ }
+ headers: {
+ 'x-ms-command-name': 'FinOpsToolkit.Hubs.config_RunExportJobs.CostsMonthly@${finOpsToolkitVersion}'
+ ClientType: 'FinOpsToolkit.Hubs@${finOpsToolkitVersion}'
+ }
+ authentication: {
+ type: 'MSI'
+ resource: {
+ value: '@variables(\'resourceManagementUri\')'
+ type: 'Expression'
+ }
+ }
+ }
+ }
+ ]
+ }
+ { // MCA
+ value: 'mca'
+ activities: [
+ {
+ name: 'Export Type Unsupported Error'
+ type: 'Fail'
+ dependsOn: []
+ userProperties: []
+ typeProperties: {
+ message: {
+ value: '@concat(\'MCA agreements are not supported for managed exports :\',variables(\'exportScope\'))'
+ type: 'Expression'
+ }
+ errorCode: 'ExportTypeUnsupported'
+ }
+ }
+ ]
+ }
+ ]
+ defaultActivities: [
+ {
+ name: 'Export Type Not Defined Error'
+ type: 'Fail'
+ dependsOn: []
+ userProperties: []
+ typeProperties: {
+ message: {
+ value: '@concat(\'Unable to determine the export scope type for :\',variables(\'exportScope\'))'
+ type: 'Expression'
+ }
+ errorCode: 'ExportTypeNotDefined'
+ }
+ }
+ ]
+ }
+ }
+ ]
+ }
+ }
+ ]
+ concurrency: 1
+ variables: {
+ scopesArray: {
+ type: 'Array'
+ }
+ exportName: {
+ type: 'String'
+ }
+ exportScope: {
+ type: 'String'
+ }
+ exportScopeType: {
+ type: 'String'
+ }
+ storageAccountId: {
+ type: 'String'
+ defaultValue: storageAccount.id
+ }
+ finOpsHub: {
+ type: 'String'
+ defaultValue: app.hub.name
+ }
+ resourceManagementUri: {
+ type: 'String'
+ defaultValue: environment().resourceManager
+ }
+ fileName: {
+ type: 'String'
+ defaultValue: 'settings.json'
+ }
+ folderPath: {
+ type: 'String'
+ defaultValue: CONFIG
+ }
+ }
+ }
+ }
+}
+
+// TODO: Can we move this into hub-types.bicep or merge it here?
+module timeZones 'timeZones.bicep' = {
+ name: 'Microsoft.CostManagement.ManagedExports_TimeZones'
+ params: {
+ location: app.hub.location
+ }
+}
+
+module trigger_SettingsUpdated '../../fx/hub-eventTrigger.bicep' = {
+ name: 'Microsoft.FinOpsHubs.Core_SettingsUpdatedTrigger'
+ params: {
+ dataFactoryName: dataFactory.name
+ triggerName: '${CONFIG}_SettingsUpdated'
+
+ // TODO: Replace pipeline with event: 'Microsoft.FinOpsHubs.Core.SettingsUpdated'
+ pipelineName: dataFactory::pipeline_ConfigureExports.name
+ pipelineParameters: {}
+
+ storageAccountName: app.storage
+ storageContainer: CONFIG
+ // TODO: Change this to startswith
+ storagePathEndsWith: 'settings.json'
+ }
+}
+
+
+//==============================================================================
+// Outputs
+//==============================================================================
+
+// None
diff --git a/src/templates/finops-hub/modules/azuretimezones.bicep b/src/templates/finops-hub/modules/Microsoft.CostManagement/ManagedExports/timeZones.bicep
similarity index 100%
rename from src/templates/finops-hub/modules/azuretimezones.bicep
rename to src/templates/finops-hub/modules/Microsoft.CostManagement/ManagedExports/timeZones.bicep
diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep
new file mode 100644
index 000000000..6e290e2da
--- /dev/null
+++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep
@@ -0,0 +1,1909 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { finOpsToolkitVersion, HubAppProperties, privateRoutingForLinkedServices } from '../../fx/hub-types.bicep'
+
+
+//==============================================================================
+// Parameters
+//==============================================================================
+
+@description('Required. FinOps hub app getting deployed.')
+param app HubAppProperties
+
+@description('Optional. Name of the Azure Data Explorer cluster to use for advanced analytics. If empty, Azure Data Explorer will not be deployed. Required to use with Power BI if you have more than $2-5M/mo in costs being monitored. Default: "" (do not use).')
+@maxLength(22)
+param clusterName string = ''
+
+// https://learn.microsoft.com/azure/templates/microsoft.kusto/clusters?pivots=deployment-language-bicep#azuresku
+@description('Optional. Name of the Azure Data Explorer SKU. Default: "Dev(No SLA)_Standard_E2a_v4".')
+@allowed([
+ 'Dev(No SLA)_Standard_E2a_v4' // 2 CPU, 16GB RAM, 24GB cache, $110/mo
+ 'Dev(No SLA)_Standard_D11_v2' // 2 CPU, 14GB RAM, 78GB cache, $121/mo
+ 'Standard_D11_v2' // 2 CPU, 14GB RAM, 78GB cache, $245/mo
+ 'Standard_D12_v2'
+ 'Standard_D13_v2'
+ 'Standard_D14_v2'
+ 'Standard_D16d_v5'
+ 'Standard_D32d_v4'
+ 'Standard_D32d_v5'
+ 'Standard_DS13_v2+1TB_PS'
+ 'Standard_DS13_v2+2TB_PS'
+ 'Standard_DS14_v2+3TB_PS'
+ 'Standard_DS14_v2+4TB_PS'
+ 'Standard_E2a_v4' // 2 CPU, 14GB RAM, 78GB cache, $220/mo
+ 'Standard_E2ads_v5'
+ 'Standard_E2d_v4'
+ 'Standard_E2d_v5'
+ 'Standard_E4a_v4'
+ 'Standard_E4ads_v5'
+ 'Standard_E4d_v4'
+ 'Standard_E4d_v5'
+ 'Standard_E8a_v4'
+ 'Standard_E8ads_v5'
+ 'Standard_E8as_v4+1TB_PS'
+ 'Standard_E8as_v4+2TB_PS'
+ 'Standard_E8as_v5+1TB_PS'
+ 'Standard_E8as_v5+2TB_PS'
+ 'Standard_E8d_v4'
+ 'Standard_E8d_v5'
+ 'Standard_E8s_v4+1TB_PS'
+ 'Standard_E8s_v4+2TB_PS'
+ 'Standard_E8s_v5+1TB_PS'
+ 'Standard_E8s_v5+2TB_PS'
+ 'Standard_E16a_v4'
+ 'Standard_E16ads_v5'
+ 'Standard_E16as_v4+3TB_PS'
+ 'Standard_E16as_v4+4TB_PS'
+ 'Standard_E16as_v5+3TB_PS'
+ 'Standard_E16as_v5+4TB_PS'
+ 'Standard_E16d_v4'
+ 'Standard_E16d_v5'
+ 'Standard_E16s_v4+3TB_PS'
+ 'Standard_E16s_v4+4TB_PS'
+ 'Standard_E16s_v5+3TB_PS'
+ 'Standard_E16s_v5+4TB_PS'
+ 'Standard_E64i_v3'
+ 'Standard_E80ids_v4'
+ 'Standard_EC8ads_v5'
+ 'Standard_EC8as_v5+1TB_PS'
+ 'Standard_EC8as_v5+2TB_PS'
+ 'Standard_EC16ads_v5'
+ 'Standard_EC16as_v5+3TB_PS'
+ 'Standard_EC16as_v5+4TB_PS'
+ 'Standard_L4s'
+ 'Standard_L8as_v3'
+ 'Standard_L8s'
+ 'Standard_L8s_v2'
+ 'Standard_L8s_v3'
+ 'Standard_L16as_v3'
+ 'Standard_L16s'
+ 'Standard_L16s_v2'
+ 'Standard_L16s_v3'
+ 'Standard_L32as_v3'
+ 'Standard_L32s_v3'
+])
+param clusterSku string = 'Dev(No SLA)_Standard_E2a_v4'
+
+@description('Optional. Number of nodes to use in the cluster. Allowed values: 1 for the Basic SKU tier and 2-1000 for Standard. Default: 1 for dev/test SKUs, 2 for standard SKUs.')
+@minValue(1)
+@maxValue(1000)
+param clusterCapacity int = 1
+
+// TODO: Figure out why this is breaking upgrades
+// @description('Optional. Array of external tenant IDs that should have access to the cluster. Default: empty (no external access).')
+// param clusterTrustedExternalTenants string[] = []
+
+// cSpell:ignore eventhouse
+@description('Optional. Microsoft Fabric eventhouse query URI. Default: "" (do not use).')
+param fabricQueryUri string = ''
+
+@description('Optional. Number of capacity units for the Microsoft Fabric capacity. This is the number in your Fabric SKU (e.g., Trial = 1, F2 = 2, F64 = 64). This is used to manage parallelization in data pipelines. If you change capacity, please redeploy the template. Allowed values: 1 for the Fabric trial and 2-2048 based on the assigned Fabric capacity (e.g., F2-F2048). Default: 2.')
+@minValue(1)
+@maxValue(2048)
+param fabricCapacityUnits int = 2
+
+@description('Optional. Forces the table to be updated if different from the last time it was deployed.')
+param forceUpdateTag string = utcNow()
+
+@description('Optional. If true, ingestion will continue even if some rows fail to ingest. Default: false.')
+param continueOnErrors bool = false
+
+@description('Required. Number of days of data to retain in the Data Explorer *_raw tables.')
+param rawRetentionInDays int
+
+
+//==============================================================================
+// Variables
+//==============================================================================
+
+var CONFIG = 'config'
+var HUB_DATA_EXPLORER = 'hubDataExplorer'
+var HUB_DB = 'Hub'
+var INGESTION = 'ingestion'
+var INGESTION_DB = 'Ingestion'
+var INGESTION_ID_SEPARATOR = '__'
+
+var ftkReleaseUri = endsWith(finOpsToolkitVersion, '-dev')
+ ? 'https://github.com/microsoft/finops-toolkit/releases/latest/download'
+ : 'https://github.com/microsoft/finops-toolkit/releases/download/v${finOpsToolkitVersion}'
+
+var useFabric = !empty(fabricQueryUri)
+var useAzure = !useFabric && !empty(clusterName)
+
+// cSpell:ignore ftkver, privatelink
+var dataExplorerPrivateDnsZoneName = replace('privatelink.${app.hub.location}.${replace(environment().suffixes.storage, 'core', 'kusto')}', '..', '.')
+
+// Actual = Minimum(ClusterMaximumConcurrentOperations, Number of nodes in cluster * Maximum(1, Core count per node * CoreUtilizationCoefficient))
+var ingestionCapacity = {
+ 'Dev(No SLA)_Standard_E2a_v4': 1
+ 'Dev(No SLA)_Standard_D11_v2': 1
+ Standard_D11_v2: 2
+ Standard_D12_v2: 4
+ Standard_D13_v2: 8
+ Standard_D14_v2: 16
+ Standard_D16d_v5: 16
+ Standard_D32d_v4: 32
+ Standard_D32d_v5: 32
+ 'Standard_DS13_v2+1TB_PS': 8
+ 'Standard_DS13_v2+2TB_PS': 8
+ 'Standard_DS14_v2+3TB_PS': 16
+ 'Standard_DS14_v2+4TB_PS': 16
+ Standard_E2a_v4: 2
+ Standard_E2ads_v5: 2
+ Standard_E2d_v4: 2
+ Standard_E2d_v5: 2
+ Standard_E4a_v4: 4
+ Standard_E4ads_v5: 4
+ Standard_E4d_v4: 4
+ Standard_E4d_v5: 4
+ Standard_E8a_v4: 8
+ Standard_E8ads_v5: 8
+ 'Standard_E8as_v4+1TB_PS': 8
+ 'Standard_E8as_v4+2TB_PS': 8
+ 'Standard_E8as_v5+1TB_PS': 8
+ 'Standard_E8as_v5+2TB_PS': 8
+ Standard_E8d_v4: 8
+ Standard_E8d_v5: 8
+ 'Standard_E8s_v4+1TB_PS': 8
+ 'Standard_E8s_v4+2TB_PS': 8
+ 'Standard_E8s_v5+1TB_PS': 8
+ 'Standard_E8s_v5+2TB_PS': 8
+ Standard_E16a_v4: 16
+ Standard_E16ads_v5: 16
+ 'Standard_E16as_v4+3TB_PS': 16
+ 'Standard_E16as_v4+4TB_PS': 16
+ 'Standard_E16as_v5+3TB_PS': 16
+ 'Standard_E16as_v5+4TB_PS': 16
+ Standard_E16d_v4: 16
+ Standard_E16d_v5: 16
+ 'Standard_E16s_v4+3TB_PS': 16
+ 'Standard_E16s_v4+4TB_PS': 16
+ 'Standard_E16s_v5+3TB_PS': 16
+ 'Standard_E16s_v5+4TB_PS': 16
+ Standard_E64i_v3: 64
+ Standard_E80ids_v4: 80
+ Standard_EC8ads_v5: 8
+ 'Standard_EC8as_v5+1TB_PS': 8
+ 'Standard_EC8as_v5+2TB_PS': 8
+ Standard_EC16ads_v5: 16
+ 'Standard_EC16as_v5+3TB_PS': 16
+ 'Standard_EC16as_v5+4TB_PS': 16
+ Standard_L4s: 4
+ Standard_L8as_v3: 8
+ Standard_L8s: 8
+ Standard_L8s_v2: 8
+ Standard_L8s_v3: 8
+ Standard_L16as_v3: 16
+ Standard_L16s: 16
+ Standard_L16s_v2: 16
+ Standard_L16s_v3: 16
+ Standard_L32as_v3: 32
+ Standard_L32s_v3: 32
+}
+
+var dataExplorerIngestionCapacity = useFabric
+ ? fabricCapacityUnits
+ : (!useAzure ? 1 : ingestionCapacity[?clusterSku] ?? 1)
+
+// WORKAROUND: Direct property access fails on cluster updates due to ARM bug
+// See: https://github.com/Azure/azure-resource-manager-templates/issues/[issue-number]
+var dataExplorerUri = useFabric ? fabricQueryUri : 'https://${cluster.name}.${app.hub.location}.kusto.windows.net'
+
+//==============================================================================
+// Resources
+//==============================================================================
+
+// App registration
+module appRegistration '../../fx/hub-app.bicep' = {
+ name: 'Microsoft.FinOpsHubs.Analytics_Register'
+ params: {
+ app: app
+ version: finOpsToolkitVersion
+ features: [
+ 'DataFactory'
+ 'Storage'
+ ]
+ }
+}
+
+//------------------------------------------------------------------------------
+// Dependencies
+//------------------------------------------------------------------------------
+
+// Get data factory instance
+resource dataFactory 'Microsoft.DataFactory/factories@2018-06-01' existing = {
+ name: app.dataFactory
+ dependsOn: [
+ appRegistration
+ ]
+}
+
+resource blobPrivateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' existing = {
+ name: 'privatelink.blob.${environment().suffixes.storage}'
+ dependsOn: [
+ appRegistration
+ ]
+}
+
+resource queuePrivateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' existing = {
+ name: 'privatelink.queue.${environment().suffixes.storage}'
+ dependsOn: [
+ appRegistration
+ ]
+}
+
+resource tablePrivateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' existing = {
+ name: 'privatelink.table.${environment().suffixes.storage}'
+ dependsOn: [
+ appRegistration
+ ]
+}
+
+resource storage 'Microsoft.Storage/storageAccounts@2022-09-01' existing = {
+ name: app.storage
+ dependsOn: [
+ appRegistration
+ ]
+}
+
+//------------------------------------------------------------------------------
+// Cluster + databases
+//------------------------------------------------------------------------------
+
+// Kusto cluster
+resource cluster 'Microsoft.Kusto/clusters@2023-08-15' = if (useAzure) {
+ name: replace(clusterName, '_', '-')
+ dependsOn: [
+ appRegistration
+ ]
+ location: app.hub.location
+ tags: union(app.tags, app.hub.tagsByResource[?'Microsoft.Kusto/clusters'] ?? {})
+ sku: {
+ name: clusterSku
+ tier: startsWith(clusterSku, 'Dev(No SLA)_') ? 'Basic' : 'Standard'
+ capacity: startsWith(clusterSku, 'Dev(No SLA)_') ? 1 : (clusterCapacity == 1 ? 2 : clusterCapacity)
+ }
+ identity: {
+ type: 'SystemAssigned'
+ }
+ properties: {
+ enableStreamingIngest: true
+ enableAutoStop: false
+ publicNetworkAccess: app.hub.options.privateRouting ? 'Disabled' : 'Enabled'
+ // TODO: Figure out why this is breaking upgrades
+ // trustedExternalTenants: [for tenantId in clusterTrustedExternalTenants: {
+ // value: tenantId
+ // }]
+ }
+
+ resource adfClusterAdmin 'principalAssignments' = {
+ name: 'adf-mi-cluster-admin'
+ properties: {
+ principalType: 'App'
+ principalId: dataFactory.identity.principalId
+ tenantId: dataFactory.identity.tenantId
+ role: 'AllDatabasesAdmin'
+ }
+ }
+
+ resource ingestionDb 'databases' = {
+ name: INGESTION_DB
+ location: app.hub.location
+ kind: 'ReadWrite'
+ }
+
+ resource hubDb 'databases' = {
+ name: HUB_DB
+ location: app.hub.location
+ kind: 'ReadWrite'
+ }
+}
+
+module ingestion_OpenDataInternalScripts '../../fx/hub-database.bicep' = if (useAzure) {
+ name: 'Microsoft.FinOpsHubs.Analytics_ADX.IngestionOpenDataInternal'
+ params: {
+ clusterName: cluster.name
+ databaseName: cluster::ingestionDb.name
+ scripts: {
+ OpenDataFunctions_resource_type_1: loadTextContent('scripts/OpenDataFunctions_resource_type_1.kql')
+ OpenDataFunctions_resource_type_2: loadTextContent('scripts/OpenDataFunctions_resource_type_2.kql')
+ OpenDataFunctions_resource_type_3: loadTextContent('scripts/OpenDataFunctions_resource_type_3.kql')
+ OpenDataFunctions_resource_type_4: loadTextContent('scripts/OpenDataFunctions_resource_type_4.kql')
+ OpenDataFunctions_resource_type_5: loadTextContent('scripts/OpenDataFunctions_resource_type_5.kql')
+ }
+ continueOnErrors: continueOnErrors
+ forceUpdateTag: forceUpdateTag
+ }
+}
+
+module ingestion_InitScripts '../../fx/hub-database.bicep' = if (useAzure) {
+ name: 'Microsoft.FinOpsHubs.Analytics_ADX.IngestionInit'
+ dependsOn: [
+ ingestion_OpenDataInternalScripts
+ ]
+ params: {
+ clusterName: cluster.name
+ databaseName: cluster::ingestionDb.name
+ scripts: {
+ openData: loadTextContent('scripts/OpenDataFunctions.kql')
+ common: loadTextContent('scripts/Common.kql')
+ infra: loadTextContent('scripts/IngestionSetup_HubInfra.kql')
+ rawTables: replace(loadTextContent('scripts/IngestionSetup_RawTables.kql'), '$$rawRetentionInDays$$', string(rawRetentionInDays))
+ }
+ continueOnErrors: continueOnErrors
+ forceUpdateTag: forceUpdateTag
+ }
+}
+
+module ingestion_VersionedScripts '../../fx/hub-database.bicep' = if (useAzure) {
+ name: 'Microsoft.FinOpsHubs.Analytics_ADX.IngestionVersioned'
+ dependsOn: [
+ ingestion_InitScripts
+ ]
+ params: {
+ clusterName: cluster.name
+ databaseName: cluster::ingestionDb.name
+ scripts: {
+ v1_0: loadTextContent('scripts/IngestionSetup_v1_0.kql')
+ v1_2: loadTextContent('scripts/IngestionSetup_v1_2.kql')
+ }
+ continueOnErrors: continueOnErrors
+ forceUpdateTag: forceUpdateTag
+ }
+}
+
+module hub_InitScripts '../../fx/hub-database.bicep' = if (useAzure) {
+ name: 'Microsoft.FinOpsHubs.Analytics_ADX.HubInit'
+ dependsOn: [
+ ingestion_InitScripts
+ ]
+ params: {
+ clusterName: cluster.name
+ databaseName: cluster::hubDb.name
+ scripts: {
+ common: loadTextContent('scripts/Common.kql')
+ openData: loadTextContent('scripts/HubSetup_OpenData.kql')
+ }
+ continueOnErrors: continueOnErrors
+ forceUpdateTag: forceUpdateTag
+ }
+}
+
+module hub_VersionedScripts '../../fx/hub-database.bicep' = if (useAzure) {
+ name: 'Microsoft.FinOpsHubs.Analytics_ADX.HubVersioned'
+ dependsOn: [
+ ingestion_VersionedScripts
+ hub_InitScripts
+ ]
+ params: {
+ clusterName: cluster.name
+ databaseName: cluster::hubDb.name
+ scripts: {
+ v1_0: loadTextContent('scripts/HubSetup_v1_0.kql')
+ v1_2: loadTextContent('scripts/HubSetup_v1_2.kql')
+ }
+ continueOnErrors: continueOnErrors
+ forceUpdateTag: forceUpdateTag
+ }
+}
+
+module hub_LatestScripts '../../fx/hub-database.bicep' = if (useAzure) {
+ name: 'Microsoft.FinOpsHubs.Analytics_ADX.HubLatest'
+ dependsOn: [
+ hub_VersionedScripts
+ ]
+ params: {
+ clusterName: cluster.name
+ databaseName: cluster::hubDb.name
+ scripts: {
+ latest: loadTextContent('scripts/HubSetup_Latest.kql')
+ }
+ continueOnErrors: continueOnErrors
+ forceUpdateTag: forceUpdateTag
+ }
+}
+
+// Authorize Kusto Cluster to read storage
+resource clusterStorageAccess 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (useAzure) {
+ name: guid(cluster.name, subscription().id, 'Storage Blob Data Contributor')
+ scope: storage
+ properties: {
+ description: 'Give "Storage Blob Data Contributor" to the cluster'
+ #disable-next-line BCP318 // Null safety warning for conditional resource access // Null safety warning for conditional resource access
+ principalId: cluster.identity.principalId
+ // Required in case principal not ready when deploying the assignment
+ principalType: 'ServicePrincipal'
+ roleDefinitionId: subscriptionResourceId(
+ 'Microsoft.Authorization/roleDefinitions',
+ 'ba92f5b4-2d11-453d-a403-e96b0029c9fe' // Storage Blob Data Contributor -- https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#storage
+ )
+ }
+}
+
+// DNS zone
+resource dataExplorerPrivateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' = if (useAzure && app.hub.options.privateRouting) {
+ name: dataExplorerPrivateDnsZoneName
+ location: 'global'
+ tags: union(app.tags, app.hub.tagsByResource[?'Microsoft.Network/privateDnsZones'] ?? {})
+ properties: {}
+}
+
+// Link DNS zone to VNet
+resource dataExplorerPrivateDnsZoneLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2024-06-01' = if (useAzure && app.hub.options.privateRouting) {
+ name: '${replace(dataExplorerPrivateDnsZone.name, '.', '-')}-link'
+ location: 'global'
+ parent: dataExplorerPrivateDnsZone
+ tags: union(app.tags, app.hub.tagsByResource[?'Microsoft.Network/privateDnsZones/virtualNetworkLinks'] ?? {})
+ properties: {
+ virtualNetwork: {
+ id: app.hub.routing.networkId
+ }
+ registrationEnabled: false
+ }
+}
+
+// Private endpoint
+resource dataExplorerEndpoint 'Microsoft.Network/privateEndpoints@2023-11-01' = if (useAzure && app.hub.options.privateRouting) {
+ name: '${cluster.name}-ep'
+ location: app.hub.location
+ tags: union(app.tags, app.hub.tagsByResource[?'Microsoft.Network/privateEndpoints'] ?? {})
+ properties: {
+ subnet: {
+ id: app.hub.routing.subnets.dataExplorer
+ }
+ privateLinkServiceConnections: [
+ {
+ name: 'dataExplorerLink'
+ properties: {
+ privateLinkServiceId: cluster.id
+ groupIds: ['cluster']
+ }
+ }
+ ]
+ }
+}
+
+// DNS records for private endpoint
+resource dataExplorerPrivateDnsZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2023-11-01' = if (useAzure && app.hub.options.privateRouting) {
+ name: 'dataExplorer-endpoint-zone'
+ parent: dataExplorerEndpoint
+ properties: {
+ privateDnsZoneConfigs: [
+ {
+ name: 'privatelink-westus-kusto-net'
+ properties: {
+ privateDnsZoneId: dataExplorerPrivateDnsZone.id
+ }
+ }
+ {
+ name: 'privatelink-blob-core-windows-net'
+ properties: {
+ privateDnsZoneId: blobPrivateDnsZone.id
+ }
+ }
+ {
+ name: 'privatelink-table-core-windows-net'
+ properties: {
+ privateDnsZoneId: tablePrivateDnsZone.id
+ }
+ }
+ {
+ name: 'privatelink-queue-core-windows-net'
+ properties: {
+ privateDnsZoneId: queuePrivateDnsZone.id
+ }
+ }
+ ]
+ }
+}
+
+//------------------------------------------------------------------------------
+// Data Factory setup
+// cSpell:ignore linkedservices
+//------------------------------------------------------------------------------
+
+resource dataFactoryVNet 'Microsoft.DataFactory/factories/managedVirtualNetworks@2018-06-01' existing = if (useAzure && app.hub.options.privateRouting) {
+ name: 'default'
+ parent: dataFactory
+
+ resource dataExplorerManagedPrivateEndpoint 'managedPrivateEndpoints' = {
+ name: HUB_DATA_EXPLORER
+ properties: {
+ name: HUB_DATA_EXPLORER
+ groupId: 'cluster'
+ #disable-next-line BCP318 // Null safety warning for conditional resource access // Null safety warning for conditional resource access // Null safety warning for conditional resource access
+ privateLinkResourceId: cluster.id
+ fqdns: [
+ 'https://${replace(clusterName, '_', '-')}.${app.hub.location}.kusto.windows.net'
+ ]
+ }
+ }
+}
+
+module getDataExplorerPrivateEndpointConnections 'dataExplorerEndpoints.bicep' = if (useAzure && app.hub.options.privateRouting) {
+ name: 'GetDataExplorerPrivateEndpointConnections'
+ dependsOn: [
+ dataFactoryVNet::dataExplorerManagedPrivateEndpoint
+ ]
+ params: {
+ dataExplorerName: cluster.name
+ }
+}
+
+module approveDataExplorerPrivateEndpointConnections 'dataExplorerEndpoints.bicep' = if (useAzure && app.hub.options.privateRouting) {
+ name: 'ApproveDataExplorerPrivateEndpointConnections'
+ params: {
+ #disable-next-line BCP318 // Null safety warning for conditional resource access // Null safety warning for conditional resource access
+ dataExplorerName: cluster.name
+ #disable-next-line BCP318 // Null safety warning for conditional resource access // Null safety warning for conditional resource access
+ privateEndpointConnections: getDataExplorerPrivateEndpointConnections.outputs.privateEndpointConnections
+ }
+}
+
+// ADX/Fabric linked service
+resource linkedService_dataExplorer 'Microsoft.DataFactory/factories/linkedservices@2018-06-01' = if (useAzure || useFabric) {
+ name: HUB_DATA_EXPLORER
+ parent: dataFactory
+ properties: {
+ type: 'AzureDataExplorer'
+ parameters: {
+ database: {
+ type: 'String'
+ defaultValue: INGESTION_DB
+ }
+ }
+ typeProperties: {
+ endpoint: dataExplorerUri
+ database: '@{linkedService().database}'
+ tenant: dataFactory.identity.tenantId
+ servicePrincipalId: dataFactory.identity.principalId
+ }
+ ...privateRoutingForLinkedServices(app.hub)
+ }
+}
+
+// GitHub repository linked service for FTK open data
+resource linkedService_ftkRepo 'Microsoft.DataFactory/factories/linkedservices@2018-06-01' = {
+ name: 'ftkRepo'
+ parent: dataFactory
+ properties: {
+ type: 'HttpServer'
+ parameters: {
+ filePath: {
+ type: 'string'
+ }
+ }
+ typeProperties: {
+ url: '@concat(\'https://gitapp.hub.com/microsoft/finops-toolkit/\', linkedService().filePath)'
+ enableServerCertificateValidation: true
+ authenticationType: 'Anonymous'
+ }
+ ...privateRoutingForLinkedServices(app.hub)
+ }
+}
+
+resource dataset_dataExplorer 'Microsoft.DataFactory/factories/datasets@2018-06-01' = {
+ name: HUB_DATA_EXPLORER
+ parent: dataFactory
+ properties: {
+ type: 'AzureDataExplorerTable'
+ linkedServiceName: {
+ parameters: {
+ database: '@dataset().database'
+ }
+ referenceName: linkedService_dataExplorer.name
+ type: 'LinkedServiceReference'
+ }
+ parameters: {
+ database: {
+ type: 'String'
+ defaultValue: INGESTION_DB // Do not use dynamic reference since that won't work with Fabric
+ }
+ table: { type: 'String' }
+ }
+ typeProperties: {
+ table: {
+ value: '@dataset().table'
+ type: 'Expression'
+ }
+ }
+ }
+}
+
+resource dataset_ftkReleaseFile 'Microsoft.DataFactory/factories/datasets@2018-06-01' = {
+ name: 'ftkReleaseFile'
+ parent: dataFactory
+ properties: {
+ linkedServiceName: {
+ referenceName: linkedService_ftkRepo.name
+ type: 'LinkedServiceReference'
+ }
+ parameters: {
+ fileName: {
+ type: 'string'
+ }
+ version: {
+ type: 'string'
+ defaultValue: finOpsToolkitVersion
+ }
+ }
+ annotations: []
+ type: 'DelimitedText'
+ typeProperties: {
+ location: {
+ type: 'HttpServerLocation'
+ relativeUrl: {
+ value: '@concat(\'releases/download/v\', dataset().version, \'/\', dataset().fileName)'
+ type: 'Expression'
+ }
+ }
+ columnDelimiter: ','
+ escapeChar: '\\'
+ firstRowAsHeader: true
+ quoteChar: '"'
+ }
+ schema: []
+ }
+}
+
+module trigger_IngestionManifestAdded '../../fx/hub-eventTrigger.bicep' = {
+ name: 'Microsoft.FinOpsHubs.Core_IngestionManifestAddedTrigger'
+ params: {
+ dataFactoryName: dataFactory.name
+ triggerName: '${INGESTION}_ManifestAdded'
+
+ // TODO: Replace pipeline with event: 'Microsoft.FinOpsHubs.Core.IngestionManifestAdded'
+ pipelineName: pipeline_ExecuteIngestionETL.name
+ pipelineParameters: {
+ folderPath: '@triggerBody().folderPath'
+ }
+
+ storageAccountName: app.storage
+ storageContainer: INGESTION
+ storagePathEndsWith: 'manifest.json'
+ }
+}
+
+//------------------------------------------------------------------------------
+// config_InitializeHub pipeline
+//------------------------------------------------------------------------------
+@description('Initializes the hub instance based on the configuration settings.')
+resource pipeline_InitializeHub 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = {
+ name: '${CONFIG}_InitializeHub'
+ parent: dataFactory
+ properties: {
+ activities: [
+ { // Get Config
+ name: 'Get Config'
+ type: 'Lookup'
+ dependsOn: []
+ policy: {
+ timeout: '0.00:05:00'
+ retry: 2
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ source: {
+ type: 'JsonSource'
+ storeSettings: {
+ type: 'AzureBlobFSReadSettings'
+ recursive: true
+ enablePartitionDiscovery: false
+ }
+ formatSettings: {
+ type: 'JsonReadSettings'
+ }
+ }
+ dataset: {
+ referenceName: CONFIG
+ type: 'DatasetReference'
+ }
+ }
+ }
+ { // Set Version
+ name: 'Set Version'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Get Config'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ variableName: 'version'
+ value: {
+ value: '@activity(\'Get Config\').output.firstRow.version'
+ type: 'Expression'
+ }
+ }
+ }
+ { // Set Scopes
+ name: 'Set Scopes'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Get Config'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ variableName: 'scopes'
+ value: {
+ value: '@string(activity(\'Get Config\').output.firstRow.scopes)'
+ type: 'Expression'
+ }
+ }
+ }
+ { // Set Retention
+ name: 'Set Retention'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Get Config'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ variableName: 'retention'
+ value: {
+ value: '@string(activity(\'Get Config\').output.firstRow.retention)'
+ type: 'Expression'
+ }
+ }
+ }
+ { // Until Capacity Is Available
+ name: 'Until Capacity Is Available'
+ type: 'Until'
+ dependsOn: [
+ {
+ activity: 'Set Version'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ {
+ activity: 'Set Scopes'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ {
+ activity: 'Set Retention'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ expression: {
+ value: '@equals(variables(\'tryAgain\'), false)'
+ type: 'Expression'
+ }
+ activities: [
+ { // Confirm Ingestion Capacity
+ name: 'Confirm Ingestion Capacity'
+ type: 'AzureDataExplorerCommand'
+ dependsOn: []
+ policy: {
+ timeout: '0.12:00:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ // cSpell:ignore Ingestions
+ command: '.show capacity | where Resource == \'Ingestions\' | project Remaining'
+ commandTimeout: '00:20:00'
+ }
+ linkedServiceName: {
+ referenceName: linkedService_dataExplorer.name
+ type: 'LinkedServiceReference'
+ parameters: {
+ database: INGESTION_DB // Do not use dynamic reference since that won't work with Fabric
+ }
+ }
+ }
+ { // If Has Capacity
+ name: 'If Has Capacity'
+ type: 'IfCondition'
+ dependsOn: [
+ {
+ activity: 'Confirm Ingestion Capacity'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ expression: {
+ value: '@or(equals(activity(\'Confirm Ingestion Capacity\').output.count, 0), greater(activity(\'Confirm Ingestion Capacity\').output.value[0].Remaining, 0))'
+ type: 'Expression'
+ }
+ ifFalseActivities: [
+ { // Wait for Ingestion
+ name: 'Wait for Ingestion'
+ type: 'Wait'
+ dependsOn: []
+ userProperties: []
+ typeProperties: {
+ waitTimeInSeconds: 15
+ }
+ }
+ { // Try Again
+ name: 'Try Again'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Wait for Ingestion'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'tryAgain'
+ value: true
+ }
+ }
+ ]
+ ifTrueActivities: [
+ { // Save ingestion policy in ADX
+ name: 'Set ingestion policy in ADX'
+ type: 'AzureDataExplorerCommand'
+ dependsOn: []
+ policy: {
+ timeout: '0.12:00:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ command: {
+ // Do not attempt to set the ingestion policy if using Fabric; use a simple query as a placeholder
+ value: useFabric
+ ? '.show database ${INGESTION_DB} policy managed_identity'
+ #disable-next-line BCP318 // Null safety warning for conditional resource access // Null safety warning for conditional resource access // Null safety warning for conditional resource access // Null safety warning for conditional resource access // Null safety warning for conditional resource access
+ : '.alter-merge database ${INGESTION_DB} policy managed_identity "[ { \'ObjectId\' : \'${cluster.identity.principalId}\', \'AllowedUsages\' : \'NativeIngestion\' }]"'
+ type: 'Expression'
+ }
+ commandTimeout: '00:20:00'
+ }
+ linkedServiceName: {
+ referenceName: linkedService_dataExplorer.name
+ type: 'LinkedServiceReference'
+ parameters: {
+ database: INGESTION_DB // Do not use dynamic reference since that won't work with Fabric
+ }
+ }
+ }
+ { // Save Hub Settings in ADX
+ name: 'Save Hub Settings in ADX'
+ type: 'AzureDataExplorerCommand'
+ dependsOn: [
+ {
+ activity: 'Set ingestion policy in ADX'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ timeout: '0.12:00:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ command: {
+ // cSpell:ignore isnull, isnotempty
+ value: '@concat(\'.append HubSettingsLog <| print version="\', variables(\'version\'), \'",scopes=dynamic(\', variables(\'scopes\'), \'),retention=dynamic(\', variables(\'retention\'), \') | extend scopes = iff(isnull(scopes[0]), pack_array(scopes), scopes) | mv-apply scopeObj = scopes on (where isnotempty(scopeObj.scope) | summarize scopes = make_set(scopeObj.scope))\')'
+ type: 'Expression'
+ }
+ commandTimeout: '00:20:00'
+ }
+ linkedServiceName: {
+ referenceName: linkedService_dataExplorer.name
+ type: 'LinkedServiceReference'
+ parameters: {
+ database: INGESTION_DB // Do not use dynamic reference since that won't work with Fabric
+ }
+ }
+ }
+ { // Update PricingUnits in ADX
+ name: 'Update PricingUnits in ADX'
+ type: 'AzureDataExplorerCommand'
+ dependsOn: [
+ {
+ activity: 'Save Hub Settings in ADX'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ timeout: '0.12:00:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ // cSpell:ignore externaldata
+ command: '.set-or-replace PricingUnits <| externaldata(x_PricingUnitDescription: string, AccountTypes: string, x_PricingBlockSize: decimal, PricingUnit: string)[@"${ftkReleaseUri}/PricingUnits.csv"] with (format="csv", ignoreFirstRecord=true) | project-away AccountTypes'
+ commandTimeout: '00:20:00'
+ }
+ linkedServiceName: {
+ referenceName: linkedService_dataExplorer.name
+ type: 'LinkedServiceReference'
+ parameters: {
+ database: INGESTION_DB // Do not use dynamic reference since that won't work with Fabric
+ }
+ }
+ }
+ { // Update Regions in ADX
+ name: 'Update Regions in ADX'
+ type: 'AzureDataExplorerCommand'
+ dependsOn: [
+ {
+ activity: 'Update PricingUnits in ADX'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ timeout: '0.12:00:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ command: '.set-or-replace Regions <| externaldata(ResourceLocation: string, RegionId: string, RegionName: string)[@"${ftkReleaseUri}/Regions.csv"] with (format="csv", ignoreFirstRecord=true)'
+ commandTimeout: '00:20:00'
+ }
+ linkedServiceName: {
+ referenceName: linkedService_dataExplorer.name
+ type: 'LinkedServiceReference'
+ parameters: {
+ database: INGESTION_DB // Do not use dynamic reference since that won't work with Fabric
+ }
+ }
+ }
+ { // Update ResourceTypes in ADX
+ name: 'Update ResourceTypes in ADX'
+ type: 'AzureDataExplorerCommand'
+ dependsOn: [
+ {
+ activity: 'Update Regions in ADX'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ timeout: '0.12:00:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ command: '.set-or-replace ResourceTypes <| externaldata(x_ResourceType: string, SingularDisplayName: string, PluralDisplayName: string, LowerSingularDisplayName: string, LowerPluralDisplayName: string, IsPreview: bool, Description: string, IconUri: string, Links: string)[@"${ftkReleaseUri}/ResourceTypes.csv"] with (format="csv", ignoreFirstRecord=true) | project-away Links'
+ commandTimeout: '00:20:00'
+ }
+ linkedServiceName: {
+ referenceName: linkedService_dataExplorer.name
+ type: 'LinkedServiceReference'
+ parameters: {
+ database: INGESTION_DB // Do not use dynamic reference since that won't work with Fabric
+ }
+ }
+ }
+ { // Update Services in ADX
+ name: 'Update Services in ADX'
+ type: 'AzureDataExplorerCommand'
+ dependsOn: [
+ {
+ activity: 'Update ResourceTypes in ADX'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ timeout: '0.12:00:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ command: '.set-or-replace Services <| externaldata(x_ConsumedService: string, x_ResourceType: string, ServiceName: string, ServiceCategory: string, ServiceSubcategory: string, PublisherName: string, x_PublisherCategory: string, x_Environment: string, x_ServiceModel: string)[@"${ftkReleaseUri}/Services.csv"] with (format="csv", ignoreFirstRecord=true)'
+ commandTimeout: '00:20:00'
+ }
+ linkedServiceName: {
+ referenceName: linkedService_dataExplorer.name
+ type: 'LinkedServiceReference'
+ parameters: {
+ database: INGESTION_DB // Do not use dynamic reference since that won't work with Fabric
+ }
+ }
+ }
+ { // Ingestion Complete
+ name: 'Ingestion Complete'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Update Services in ADX'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'tryAgain'
+ value: false
+ }
+ }
+ ]
+ }
+ }
+ { // Abort On Error
+ name: 'Abort On Error'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'If Has Capacity'
+ dependencyConditions: [
+ 'Failed'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'tryAgain'
+ value: false
+ }
+ }
+ ]
+ timeout: '0.02:00:00'
+ }
+ }
+ { // Timeout Error
+ name: 'Timeout Error'
+ type: 'Fail'
+ dependsOn: [
+ {
+ activity: 'Until Capacity Is Available'
+ dependencyConditions: [
+ 'Failed'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ message: 'Data Explorer ingestion timed out after 2 hours while waiting for available capacity. Please re-run this pipeline to re-attempt ingestion. If you continue to see this error, please report an issue at https://aka.ms/ftk/ideas.'
+ errorCode: 'DataExplorerIngestionTimeout'
+ }
+ }
+ ]
+ concurrency: 1
+ variables: {
+ version: {
+ type: 'String'
+ }
+ scopes: {
+ type: 'String'
+ }
+ retention: {
+ type: 'String'
+ }
+ tryAgain: {
+ type: 'Boolean'
+ defaultValue: true
+ }
+ }
+ }
+}
+
+//------------------------------------------------------------------------------
+// ingestion_ETL_dataExplorer pipeline
+// Triggered by ingestion_ExecuteETL
+//------------------------------------------------------------------------------
+@description('Ingests parquet data into an Azure Data Explorer cluster.')
+resource pipeline_ToDataExplorer 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = if (useAzure || useFabric) {
+ name: '${INGESTION}_ETL_dataExplorer'
+ parent: dataFactory
+ properties: {
+ activities: [
+ { // Read Hub Config
+ name: 'Read Hub Config'
+ description: 'Read the hub config to determine how long data should be retained.'
+ type: 'Lookup'
+ policy: {
+ timeout: '0.12:00:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ source: {
+ type: 'JsonSource'
+ storeSettings: {
+ type: 'AzureBlobFSReadSettings'
+ recursive: false
+ enablePartitionDiscovery: false
+ }
+ formatSettings: {
+ type: 'JsonReadSettings'
+ }
+ }
+ dataset: {
+ referenceName: CONFIG
+ type: 'DatasetReference'
+ parameters: {
+ fileName: 'settings.json'
+ folderPath: CONFIG
+ }
+ }
+ }
+ }
+ { // Set Final Retention Months
+ name: 'Set Final Retention Months'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Read Hub Config'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'finalRetentionMonths'
+ value: {
+ value: '@coalesce(activity(\'Read Hub Config\').output.firstRow.retention.final.months, 999)'
+ type: 'Expression'
+ }
+ }
+ }
+ { // Until Capacity Is Available
+ name: 'Until Capacity Is Available'
+ type: 'Until'
+ dependsOn: [
+ {
+ activity: 'Set Final Retention Months'
+ dependencyConditions: [
+ 'Completed'
+ 'Skipped'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ expression: {
+ value: '@equals(variables(\'tryAgain\'), false)'
+ type: 'Expression'
+ }
+ activities: [
+ { // Confirm Ingestion Capacity
+ name: 'Confirm Ingestion Capacity'
+ type: 'AzureDataExplorerCommand'
+ dependsOn: []
+ policy: {
+ timeout: '0.12:00:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ command: '.show capacity | where Resource == \'Ingestions\' | project Remaining'
+ commandTimeout: '00:20:00'
+ }
+ linkedServiceName: {
+ referenceName: linkedService_dataExplorer.name
+ type: 'LinkedServiceReference'
+ }
+ }
+ { // If Has Capacity
+ name: 'If Has Capacity'
+ type: 'IfCondition'
+ dependsOn: [
+ {
+ activity: 'Confirm Ingestion Capacity'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ expression: {
+ value: '@or(equals(activity(\'Confirm Ingestion Capacity\').output.count, 0), greater(activity(\'Confirm Ingestion Capacity\').output.value[0].Remaining, 0))'
+ type: 'Expression'
+ }
+ ifFalseActivities: [
+ { // Wait for Ingestion
+ name: 'Wait for Ingestion'
+ type: 'Wait'
+ dependsOn: []
+ userProperties: []
+ typeProperties: {
+ waitTimeInSeconds: 15
+ }
+ }
+ { // Try Again
+ name: 'Try Again'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Wait for Ingestion'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'tryAgain'
+ value: true
+ }
+ }
+ ]
+ ifTrueActivities: [
+ { // Pre-Ingest Cleanup
+ name: 'Pre-Ingest Cleanup'
+ description: 'Cost Management exports include all month-to-date data from the previous export run. To ensure data is not double-reported, it must be dropped from the raw table before ingestion completes. Remove previous ingestions into the raw table for the month and any previous runs of the current ingestion month file in any table.'
+ type: 'AzureDataExplorerCommand'
+ dependsOn: []
+ policy: {
+ timeout: '0.12:00:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ typeProperties: {
+ command: {
+ value: '@concat(\'.drop extents <| .show extents | where (TableName == "\', pipeline().parameters.table, \'" and Tags !has "drop-by:\', pipeline().parameters.ingestionId, \'" and Tags has "drop-by:\', pipeline().parameters.folderPath, \'") or (Tags has "drop-by:\', pipeline().parameters.ingestionId, \'" and Tags has "drop-by:\', pipeline().parameters.folderPath, \'/\', pipeline().parameters.originalFileName, \'")\')'
+ type: 'Expression'
+ }
+ commandTimeout: '00:20:00'
+ }
+ linkedServiceName: {
+ referenceName: linkedService_dataExplorer.name
+ type: 'LinkedServiceReference'
+ parameters: {
+ database: INGESTION_DB // Do not use dynamic reference since that won't work with Fabric
+ }
+ }
+ }
+ { // Ingest Data
+ name: 'Ingest Data'
+ type: 'AzureDataExplorerCommand'
+ dependsOn: [
+ {
+ activity: 'Pre-Ingest Cleanup'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ timeout: '0.12:00:00'
+ retry: 3
+ retryIntervalInSeconds: 120
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ command: {
+ // cSpell:ignore abfss, toscalar
+ value: '@concat(\'.ingest into table \', pipeline().parameters.table, \' ("abfss://${INGESTION}@${app.storage}.dfs.${environment().suffixes.storage}/\', pipeline().parameters.folderPath, \'/\', pipeline().parameters.fileName, \';${useFabric ? 'impersonate' : 'managed_identity=system'}") with (format="parquet", ingestionMappingReference="\', pipeline().parameters.table, \'_mapping", tags="[\\"drop-by:\', pipeline().parameters.ingestionId, \'\\", \\"drop-by:\', pipeline().parameters.folderPath, \'/\', pipeline().parameters.originalFileName, \'\\", \\"drop-by:ftk-version-${finOpsToolkitVersion}\\"]"); print Success = assert(iff(toscalar($command_results | project-keep HasErrors) == false, true, false), "Ingestion Failed")\')'
+ type: 'Expression'
+ }
+ commandTimeout: '01:00:00'
+ }
+ linkedServiceName: {
+ referenceName: linkedService_dataExplorer.name
+ type: 'LinkedServiceReference'
+ parameters: {
+ database: INGESTION_DB // Do not use dynamic reference since that won't work with Fabric
+ }
+ }
+ }
+ { // Post-Ingest Cleanup
+ name: 'Post-Ingest Cleanup'
+ description: 'Cost Management exports include all month-to-date data from the previous export run. To ensure data is not double-reported, it must be dropped after ingestion completes. Remove the current ingestion month file from raw and any old ingestions for the month from the final table.'
+ type: 'AzureDataExplorerCommand'
+ dependsOn: [
+ {
+ activity: 'Ingest Data'
+ dependencyConditions: [
+ 'Completed'
+ ]
+ }
+ ]
+ policy: {
+ timeout: '0.12:00:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ typeProperties: {
+ command: {
+ // cSpell:ignore startofmonth, strcat, todatetime
+ value: '@concat(\'.drop extents <| .show extents | extend isOldFinalData = (TableName startswith "\', replace(pipeline().parameters.table, \'_raw\', \'_final_v\'), \'" and Tags !has "drop-by:\', pipeline().parameters.ingestionId, \'" and Tags has "drop-by:\', pipeline().parameters.folderPath, \'") | extend isPastFinalRetention = (TableName startswith "\', replace(pipeline().parameters.table, \'_raw\', \'_final_v\'), \'" and todatetime(substring(strcat(replace_string(extract("drop-by:[A-Za-z]+/(\\\\d{4}/\\\\d{2}(/\\\\d{2})?)", 1, Tags), "/", "-"), "-01"), 0, 10)) < datetime_add("month", -\', if(lessOrEquals(variables(\'finalRetentionMonths\'), 0), 0, variables(\'finalRetentionMonths\')), \', startofmonth(now()))) | where isOldFinalData or isPastFinalRetention\')'
+ type: 'Expression'
+ }
+ commandTimeout: '00:20:00'
+ }
+ linkedServiceName: {
+ referenceName: linkedService_dataExplorer.name
+ type: 'LinkedServiceReference'
+ parameters: {
+ database: INGESTION_DB // Do not use dynamic reference since that won't work with Fabric
+ }
+ }
+ }
+ { // Ingestion Complete
+ name: 'Ingestion Complete'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Post-Ingest Cleanup'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'tryAgain'
+ value: false
+ }
+ }
+ { // Abort On Ingestion Error
+ name: 'Abort On Ingestion Error'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Ingest Data'
+ dependencyConditions: [
+ 'Failed'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'tryAgain'
+ value: false
+ }
+ }
+ { // Error: DataExplorerIngestionFailed
+ name: 'Ingestion Failed Error'
+ type: 'Fail'
+ dependsOn: [
+ {
+ activity: 'Abort On Ingestion Error'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ message: {
+ value: '@concat(\'Data Explorer ingestion into the \', pipeline().parameters.table, \' table failed. Please fix the error and rerun ingestion for the following folder path: "\', pipeline().parameters.folderPath, \'". File: \', pipeline().parameters.originalFileName, \'. Error: \', if(greater(length(activity(\'Ingest Data\').output.errors), 0), activity(\'Ingest Data\').output.errors[0].Message, \'Unknown\'), \' (Code: \', if(greater(length(activity(\'Ingest Data\').output.errors), 0), activity(\'Ingest Data\').output.errors[0].Code, \'None\'), \')\')'
+ type: 'Expression'
+ }
+ errorCode: 'DataExplorerIngestionFailed'
+ }
+ }
+ { // Abort On Pre-Ingest Drop Error
+ name: 'Abort On Pre-Ingest Drop Error'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Pre-Ingest Cleanup'
+ dependencyConditions: [
+ 'Failed'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'tryAgain'
+ value: false
+ }
+ }
+ { // Error: DataExplorerPreIngestionDropFailed
+ name: 'Pre-Ingest Drop Failed Error'
+ type: 'Fail'
+ dependsOn: [
+ {
+ activity: 'Abort On Pre-Ingest Drop Error'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ message: {
+ value: '@concat(\'Data Explorer pre-ingestion cleanup (drop extents from raw table) for the \', pipeline().parameters.table, \' table failed. Ingestion was not completed. Please fix the error and rerun ingestion for the following folder path: "\', pipeline().parameters.folderPath, \'". File: \', pipeline().parameters.originalFileName, \'. Error: \', if(greater(length(activity(\'Pre-Ingest Cleanup\').output.errors), 0), activity(\'Pre-Ingest Cleanup\').output.errors[0].Message, \'Unknown\'), \' (Code: \', if(greater(length(activity(\'Pre-Ingest Cleanup\').output.errors), 0), activity(\'Pre-Ingest Cleanup\').output.errors[0].Code, \'None\'), \')\')'
+ type: 'Expression'
+ }
+ errorCode: 'DataExplorerPreIngestionDropFailed'
+ }
+ }
+ { // Abort On Post-Ingest Drop Error
+ name: 'Abort On Post-Ingest Drop Error'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Post-Ingest Cleanup'
+ dependencyConditions: [
+ 'Failed'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'tryAgain'
+ value: false
+ }
+ }
+ { // Error: DataExplorerPostIngestionDropFailed
+ name: 'Post-Ingest Drop Failed Error'
+ type: 'Fail'
+ dependsOn: [
+ {
+ activity: 'Abort On Post-Ingest Drop Error'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ message: {
+ value: '@concat(\'Data Explorer post-ingestion cleanup (drop extents from final tables) for the \', replace(pipeline().parameters.table, \'_raw\', \'_final_*\'), \' table failed. Please fix the error and rerun ingestion for the following folder path: "\', pipeline().parameters.folderPath, \'". File: \', pipeline().parameters.originalFileName, \'. Error: \', if(greater(length(activity(\'Post-Ingest Cleanup\').output.errors), 0), activity(\'Post-Ingest Cleanup\').output.errors[0].Message, \'Unknown\'), \' (Code: \', if(greater(length(activity(\'Post-Ingest Cleanup\').output.errors), 0), activity(\'Post-Ingest Cleanup\').output.errors[0].Code, \'None\'), \')\')'
+ type: 'Expression'
+ }
+ errorCode: 'DataExplorerPostIngestionDropFailed'
+ }
+ }
+ ]
+ }
+ }
+ ]
+ timeout: '0.02:00:00'
+ }
+ }
+ ]
+ parameters: {
+ folderPath: {
+ type: 'string'
+ }
+ fileName: {
+ type: 'string'
+ }
+ originalFileName: {
+ type: 'string'
+ }
+ ingestionId: {
+ type: 'string'
+ }
+ table: {
+ type: 'string'
+ }
+ }
+ variables: {
+ tryAgain: {
+ type: 'Boolean'
+ defaultValue: true
+ }
+ logRetentionDays: {
+ type: 'Integer'
+ defaultValue: 0
+ }
+ finalRetentionMonths: {
+ type: 'Integer'
+ defaultValue: 999
+ }
+ }
+ annotations: []
+ }
+}
+
+//------------------------------------------------------------------------------
+// ingestion_ExecuteETL pipeline
+// Triggered by ingestion_ManifestAdded trigger
+//------------------------------------------------------------------------------
+@description('Queues the ingestion_ETL_dataExplorer pipeline to account for Data Factory pipeline trigger limits.')
+resource pipeline_ExecuteIngestionETL 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = if (useAzure || useFabric) {
+ name: '${INGESTION}_ExecuteETL'
+ parent: dataFactory
+ properties: {
+ concurrency: 1
+ activities: [
+ { // Wait
+ name: 'Wait'
+ description: 'Files may not be available immediately after being created.'
+ type: 'Wait'
+ dependsOn: []
+ userProperties: []
+ typeProperties: {
+ waitTimeInSeconds: 60
+ }
+ }
+ { // Set Container Folder Path
+ name: 'Set Container Folder Path'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Wait'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'containerFolderPath'
+ value: {
+ value: '@join(skip(array(split(pipeline().parameters.folderPath, \'/\')), 1), \'/\')'
+ type: 'Expression'
+ }
+ }
+ }
+ { // Get Existing Parquet Files
+ name: 'Get Existing Parquet Files'
+ description: 'Get the previously ingested files so we can get file paths.'
+ type: 'GetMetadata'
+ dependsOn: [
+ {
+ activity: 'Set Container Folder Path'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ timeout: '0.12:00:00'
+ retry: 0
+ retryIntervalInSeconds: 30
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ dataset: {
+ referenceName: 'ingestion_files'
+ type: 'DatasetReference'
+ parameters: {
+ folderPath: '@variables(\'containerFolderPath\')'
+ }
+ }
+ fieldList: [
+ 'childItems'
+ ]
+ storeSettings: {
+ type: 'AzureBlobFSReadSettings'
+ enablePartitionDiscovery: false
+ }
+ formatSettings: {
+ type: 'ParquetReadSettings'
+ }
+ }
+ }
+ { // Filter Out Folders and manifest files
+ name: 'Filter Out Folders'
+ description: 'Remove any folders or manifest files.'
+ type: 'Filter'
+ dependsOn: [
+ {
+ activity: 'Get Existing Parquet Files'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ items: {
+ value: '@if(contains(activity(\'Get Existing Parquet Files\').output, \'childItems\'), activity(\'Get Existing Parquet Files\').output.childItems, json(\'[]\'))'
+ type: 'Expression'
+ }
+ condition: {
+ value: '@and(equals(item().type, \'File\'), not(contains(toLower(item().name), \'manifest.json\')))'
+ type: 'Expression'
+ }
+ }
+ }
+ { // Set Ingestion Timestamp
+ name: 'Set Ingestion Timestamp'
+ type: 'SetVariable'
+ dependsOn: [
+ {
+ activity: 'Wait'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ policy: {
+ secureOutput: false
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ variableName: 'timestamp'
+ value: {
+ value: '@utcNow()'
+ type: 'Expression'
+ }
+ }
+ }
+ { // For Each Old File
+ name: 'For Each Old File'
+ description: 'Loop thru each of the existing files.'
+ type: 'ForEach'
+ dependsOn: [
+ {
+ activity: 'Filter Out Folders'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ {
+ activity: 'Set Ingestion Timestamp'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ batchCount: dataExplorerIngestionCapacity // Concurrency limit
+ items: {
+ value: '@activity(\'Filter Out Folders\').output.Value'
+ type: 'Expression'
+ }
+ activities: [
+ { // Execute
+ name: 'Execute'
+ description: 'Run the ADX ETL pipeline.'
+ type: 'ExecutePipeline'
+ dependsOn: []
+ policy: {
+ secureInput: false
+ }
+ userProperties: []
+ typeProperties: {
+ pipeline: {
+ referenceName: pipeline_ToDataExplorer.name
+ type: 'PipelineReference'
+ }
+ waitOnCompletion: true
+ parameters: {
+ folderPath: {
+ value: '@variables(\'containerFolderPath\')'
+ type: 'Expression'
+ }
+ fileName: {
+ value: '@item().name'
+ type: 'Expression'
+ }
+ originalFileName: {
+ value: '@last(array(split(item().name, \'${INGESTION_ID_SEPARATOR}\')))'
+ type: 'Expression'
+ }
+ ingestionId: {
+ value: '@concat(first(array(split(item().name, \'${INGESTION_ID_SEPARATOR}\'))), \'_\', variables(\'timestamp\'))'
+ type: 'Expression'
+ }
+ table: {
+ value: '@concat(first(array(split(variables(\'containerFolderPath\'), \'/\'))), \'_raw\')'
+ type: 'Expression'
+ }
+ }
+ }
+ }
+ ]
+ }
+ }
+ { // If No Files
+ name: 'If No Files'
+ description: 'If there are no files found, fail the pipeline.'
+ type: 'IfCondition'
+ dependsOn: [
+ {
+ activity: 'Filter Out Folders'
+ dependencyConditions: [
+ 'Succeeded'
+ ]
+ }
+ ]
+ userProperties: []
+ typeProperties: {
+ expression: {
+ value: '@equals(length(activity(\'Filter Out Folders\').output.Value), 0)'
+ type: 'Expression'
+ }
+ ifTrueActivities: [
+ { // Error: IngestionFilesNotFound
+ name: 'Files Not Found'
+ type: 'Fail'
+ dependsOn: []
+ userProperties: []
+ typeProperties: {
+ message: {
+ value: '@concat(\'Unable to locate parquet files to ingest from the \', pipeline().parameters.folderPath, \' path. Please confirm the folder path is the full path, including the "ingestion" container and not starting with or ending with a slash ("/").\')'
+ type: 'Expression'
+ }
+ errorCode: 'IngestionFilesNotFound'
+ }
+ }
+ ]
+ }
+ }
+ ]
+ parameters: {
+ folderPath: {
+ type: 'string'
+ }
+ }
+ variables: {
+ containerFolderPath: {
+ type: 'string'
+ }
+ timestamp: {
+ type: 'string'
+ }
+ }
+ annotations: [
+ 'New ingestion'
+ ]
+ }
+}
+
+// Run initialization pipeline after everything is deployed
+module runInitializationPipeline '../../fx/hub-initialize.bicep' = if (useAzure || useFabric) {
+ name: 'Microsoft.FinOpsHubs.Analytics_InitializeHub'
+ params: {
+ app: app
+ dataFactoryInstances: [
+ app.dataFactory
+ ]
+ identityName: appRegistration.outputs.triggerManagerIdentityName
+ startPipelines: [
+ pipeline_InitializeHub.name
+ ]
+ }
+}
+
+
+//==============================================================================
+// Outputs
+//==============================================================================
+
+@description('The resource ID of the cluster.')
+#disable-next-line BCP318 // Null safety warning for conditional resource access
+output clusterId string = useFabric ? '' : cluster.id
+
+@description('The ID of the cluster system assigned managed identity.')
+#disable-next-line BCP318 // Null safety warning for conditional resource access
+output principalId string = useFabric ? '' : cluster.identity.principalId
+
+@description('The name of the cluster.')
+#disable-next-line BCP318 // Null safety warning for conditional resource access
+output clusterName string = useFabric ? '' : cluster.name
+
+@description('The URI of the cluster.')
+output clusterUri string = dataExplorerUri
+
+@description('The name of the database for data ingestion.')
+output ingestionDbName string = INGESTION_DB // Don't use cluster DB reference since that won't work for Fabric
+
+@description('The name of the database for queries.')
+output hubDbName string = HUB_DB // Don't use cluster DB reference since that won't work for Fabric
+
+@description('Max ingestion capacity of the cluster.')
+output clusterIngestionCapacity int = dataExplorerIngestionCapacity
diff --git a/src/templates/finops-hub/modules/dataExplorerEndpoints.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/dataExplorerEndpoints.bicep
similarity index 100%
rename from src/templates/finops-hub/modules/dataExplorerEndpoints.bicep
rename to src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/dataExplorerEndpoints.bicep
diff --git a/src/templates/finops-hub/modules/scripts/Common.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/Common.kql
similarity index 100%
rename from src/templates/finops-hub/modules/scripts/Common.kql
rename to src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/Common.kql
diff --git a/src/templates/finops-hub/modules/scripts/HubSetup_Latest.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql
similarity index 100%
rename from src/templates/finops-hub/modules/scripts/HubSetup_Latest.kql
rename to src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql
diff --git a/src/templates/finops-hub/modules/scripts/HubSetup_OpenData.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_OpenData.kql
similarity index 100%
rename from src/templates/finops-hub/modules/scripts/HubSetup_OpenData.kql
rename to src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_OpenData.kql
diff --git a/src/templates/finops-hub/modules/scripts/HubSetup_v1_0.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_0.kql
similarity index 100%
rename from src/templates/finops-hub/modules/scripts/HubSetup_v1_0.kql
rename to src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_0.kql
diff --git a/src/templates/finops-hub/modules/scripts/HubSetup_v1_2.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_2.kql
similarity index 100%
rename from src/templates/finops-hub/modules/scripts/HubSetup_v1_2.kql
rename to src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_2.kql
diff --git a/src/templates/finops-hub/modules/scripts/IngestionSetup_HubInfra.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_HubInfra.kql
similarity index 100%
rename from src/templates/finops-hub/modules/scripts/IngestionSetup_HubInfra.kql
rename to src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_HubInfra.kql
diff --git a/src/templates/finops-hub/modules/scripts/IngestionSetup_RawTables.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql
similarity index 100%
rename from src/templates/finops-hub/modules/scripts/IngestionSetup_RawTables.kql
rename to src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql
diff --git a/src/templates/finops-hub/modules/scripts/IngestionSetup_v1_0.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_0.kql
similarity index 100%
rename from src/templates/finops-hub/modules/scripts/IngestionSetup_v1_0.kql
rename to src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_0.kql
diff --git a/src/templates/finops-hub/modules/scripts/IngestionSetup_v1_2.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql
similarity index 100%
rename from src/templates/finops-hub/modules/scripts/IngestionSetup_v1_2.kql
rename to src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql
diff --git a/src/templates/finops-hub/modules/scripts/OpenDataFunctions.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/OpenDataFunctions.kql
similarity index 100%
rename from src/templates/finops-hub/modules/scripts/OpenDataFunctions.kql
rename to src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/OpenDataFunctions.kql
diff --git a/src/templates/finops-hub/modules/scripts/OpenDataFunctions_resource_type_1.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/OpenDataFunctions_resource_type_1.kql
similarity index 100%
rename from src/templates/finops-hub/modules/scripts/OpenDataFunctions_resource_type_1.kql
rename to src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/OpenDataFunctions_resource_type_1.kql
diff --git a/src/templates/finops-hub/modules/scripts/OpenDataFunctions_resource_type_2.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/OpenDataFunctions_resource_type_2.kql
similarity index 100%
rename from src/templates/finops-hub/modules/scripts/OpenDataFunctions_resource_type_2.kql
rename to src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/OpenDataFunctions_resource_type_2.kql
diff --git a/src/templates/finops-hub/modules/scripts/OpenDataFunctions_resource_type_3.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/OpenDataFunctions_resource_type_3.kql
similarity index 100%
rename from src/templates/finops-hub/modules/scripts/OpenDataFunctions_resource_type_3.kql
rename to src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/OpenDataFunctions_resource_type_3.kql
diff --git a/src/templates/finops-hub/modules/scripts/OpenDataFunctions_resource_type_4.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/OpenDataFunctions_resource_type_4.kql
similarity index 100%
rename from src/templates/finops-hub/modules/scripts/OpenDataFunctions_resource_type_4.kql
rename to src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/OpenDataFunctions_resource_type_4.kql
diff --git a/src/templates/finops-hub/modules/scripts/OpenDataFunctions_resource_type_5.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/OpenDataFunctions_resource_type_5.kql
similarity index 100%
rename from src/templates/finops-hub/modules/scripts/OpenDataFunctions_resource_type_5.kql
rename to src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/OpenDataFunctions_resource_type_5.kql
diff --git a/src/templates/finops-hub/modules/scripts/Copy-FileToAzureBlob.ps1 b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/Copy-FileToAzureBlob.ps1
similarity index 100%
rename from src/templates/finops-hub/modules/scripts/Copy-FileToAzureBlob.ps1
rename to src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/Copy-FileToAzureBlob.ps1
diff --git a/src/templates/finops-hub/modules/core.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/app.bicep
similarity index 50%
rename from src/templates/finops-hub/modules/core.bicep
rename to src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/app.bicep
index 46d3cb0a8..5a903c536 100644
--- a/src/templates/finops-hub/modules/core.bicep
+++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/app.bicep
@@ -1,15 +1,15 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
-import { HubProperties } from 'hub-types.bicep'
+import { finOpsToolkitVersion, HubAppProperties } from '../../fx/hub-types.bicep'
//==============================================================================
// Parameters
//==============================================================================
-@description('Required. FinOps hub instance to deploy the app to.')
-param hub HubProperties
+@description('Required. FinOps hub app getting deployed.')
+param app HubAppProperties
@description('Optional. List of scope IDs to monitor and ingest cost for.')
param scopesToMonitor array
@@ -29,75 +29,83 @@ param rawRetentionInDays int = 0
param finalRetentionInMonths int = 13
-//------------------------------------------------------------------------------
-// Temporary parameters that should be removed in the future
-//------------------------------------------------------------------------------
-
-// TODO: Consider moving telemetryString generation to hub-types.bicep
-@description('Optional. Custom string with additional metadata to log. Must an alphanumeric string without spaces or special characters except for underscores and dashes. Namespace + appName + telemetryString must be 50 characters or less - additional characters will be trimmed.')
-param telemetryString string = ''
-
-
//==============================================================================
// Variables
//==============================================================================
-var app = appRegistration.outputs.app
+var CONFIG = 'config'
+var INGESTION = 'ingestion'
//==============================================================================
// Resources
//==============================================================================
+// Networking infrastructure
+module infrastructure 'infrastructure.bicep' = {
+ name: 'Microsoft.FinOpsHubs.Core_Infrastructure'
+ params: {
+ hub: app.hub
+ }
+}
+
// Register app
-module appRegistration 'hub-app.bicep' = {
+module appRegistration '../../fx/hub-app.bicep' = {
name: 'Microsoft.FinOpsHubs.Core_Register'
+ dependsOn: [
+ infrastructure
+ ]
params: {
- hub: hub
- publisher: 'Microsoft FinOps hubs'
- namespace: 'Microsoft.FinOpsHubs'
- appName: 'Core'
- displayName: 'FinOps hub core'
- appVersion: loadTextContent('ftkver.txt') // cSpell:ignore ftkver
+ app: app
+ version: finOpsToolkitVersion
features: [
'DataFactory'
'Storage'
]
- telemetryString: telemetryString
}
}
// Create config container
-module configContainer 'hub-storage.bicep' = {
+module configContainer '../../fx/hub-storage.bicep' = {
name: 'Microsoft.FinOpsHubs.Core_Storage.ConfigContainer'
+ dependsOn: [
+ appRegistration
+ ]
params: {
app: app
- container: 'config'
+ container: CONFIG
forceCreateBlobManagerIdentity: true
}
}
// Create ingestion container
-module ingestionContainer 'hub-storage.bicep' = {
+module ingestionContainer '../../fx/hub-storage.bicep' = {
name: 'Microsoft.FinOpsHubs.Core_Storage.IngestionContainer'
+ dependsOn: [
+ appRegistration
+ ]
params: {
app: app
- container: 'ingestion'
+ container: INGESTION
}
}
// Create/update Settings.json
-module uploadSettings 'hub-deploymentScript.bicep' = {
+module uploadSettings '../../fx/hub-deploymentScript.bicep' = {
name: 'Microsoft.FinOpsHubs.Core_Storage.UpdateSettings'
+ dependsOn: [
+ appRegistration
+ ]
params: {
app: app
identityName: configContainer.outputs.identityName
scriptName: '${app.storage}_uploadSettings'
+ scriptContent: loadTextContent('Copy-FileToAzureBlob.ps1')
environmentVariables: [
{
// cSpell:ignore ftkver
name: 'ftkVersion'
- value: loadTextContent('./ftkver.txt')
+ value: finOpsToolkitVersion
}
{
name: 'scopes'
@@ -128,10 +136,107 @@ module uploadSettings 'hub-deploymentScript.bicep' = {
value: 'config'
}
]
- scriptContent: loadTextContent('./scripts/Copy-FileToAzureBlob.ps1')
}
}
+// Data Factory
+resource dataFactory 'Microsoft.DataFactory/factories@2018-06-01' existing = {
+ name: app.dataFactory
+ dependsOn: [
+ appRegistration
+ ]
+
+ // Config dataset
+ resource dataset_config 'datasets' = {
+ name: CONFIG
+ properties: {
+ linkedServiceName: {
+ referenceName: app.storage
+ type: 'LinkedServiceReference'
+ }
+ type: 'Json'
+ typeProperties: {
+ location: {
+ type: 'AzureBlobFSLocation'
+ fileName: {
+ value: '@{dataset().fileName}'
+ type: 'Expression'
+ }
+ folderPath: {
+ value: '@{dataset().folderPath}'
+ type: 'Expression'
+ }
+ }
+ }
+ parameters: {
+ fileName: {
+ type: 'String'
+ defaultValue: 'settings.json'
+ }
+ folderPath: {
+ type: 'String'
+ defaultValue: configContainer.outputs.containerName
+ }
+ }
+ }
+ }
+
+ resource dataset_ingestion 'datasets' = {
+ name: INGESTION
+ properties: {
+ annotations: []
+ parameters: {
+ blobPath: {
+ type: 'String'
+ }
+ }
+ type: 'Parquet'
+ typeProperties: {
+ location: {
+ type: 'AzureBlobFSLocation'
+ fileName: {
+ value: '@{dataset().blobPath}'
+ type: 'Expression'
+ }
+ fileSystem: ingestionContainer.outputs.containerName
+ }
+ }
+ linkedServiceName: {
+ parameters: {}
+ referenceName: app.storage
+ type: 'LinkedServiceReference'
+ }
+ }
+ }
+
+ resource dataset_ingestion_files 'datasets' = {
+ name: '${INGESTION}_files'
+ properties: {
+ annotations: []
+ parameters: {
+ folderPath: {
+ type: 'String'
+ }
+ }
+ type: 'Parquet'
+ typeProperties: {
+ location: {
+ type: 'AzureBlobFSLocation'
+ fileSystem: ingestionContainer.outputs.containerName
+ folderPath: {
+ value: '@dataset().folderPath'
+ type: 'Expression'
+ }
+ }
+ }
+ linkedServiceName: {
+ parameters: {}
+ referenceName: app.storage
+ type: 'LinkedServiceReference'
+ }
+ }
+ }
+}
//==============================================================================
// Outputs
@@ -139,24 +244,20 @@ module uploadSettings 'hub-deploymentScript.bicep' = {
// TODO: Review the use of these outputs and deprecate the ones that aren't needed, remove them in 3 months
+@description('Properties of the hub app.')
+output app HubAppProperties = app
+
@description('Name of the Data Factory.')
output dataFactoryName string = app.dataFactory
@description('Name of the storage account created for the hub instance. This must be used when connecting FinOps toolkit Power BI reports to your data.')
output storageAccountName string = app.storage
-@description('The name of the container used for configuration settings.')
-output configContainer string = configContainer.outputs.containerName
-
-@description('The name of the container used for normalized data ingestion.')
-output ingestionContainer string = ingestionContainer.outputs.containerName
-
@description('URL to use when connecting custom Power BI reports to your data.')
-output storageUrlForPowerBI string = 'https://${app.storage}.dfs.${environment().suffixes.storage}/${ingestionContainer.outputs.containerName}'
+output storageUrlForPowerBI string = 'https://${app.storage}.dfs.${environment().suffixes.storage}/${INGESTION}'
@description('Object ID of the Data Factory managed identity. This will be needed when configuring managed exports.')
-output principalId string = appRegistration.outputs.principalId
+output principalId string = dataFactory.identity.principalId
-// TODO: Remove this output
-@description('Tags for the FinOps hub publisher.')
-output publisherTags object = app.publisher.tags
+@description('Name of the managed identity used to create and stop ADF triggers.')
+output triggerManagerIdentityName string = appRegistration.outputs.triggerManagerIdentityName
diff --git a/src/templates/finops-hub/modules/infrastructure.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/infrastructure.bicep
similarity index 97%
rename from src/templates/finops-hub/modules/infrastructure.bicep
rename to src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/infrastructure.bicep
index 27e994fb0..0cf23a808 100644
--- a/src/templates/finops-hub/modules/infrastructure.bicep
+++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/infrastructure.bicep
@@ -1,7 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
-import { getHubTags, HubProperties } from 'hub-types.bicep'
+import { getHubTags, HubProperties } from '../../fx/hub-types.bicep'
//==============================================================================
@@ -293,7 +293,7 @@ resource tablePrivateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' = if
//------------------------------------------------------------------------------
resource scriptStorageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' = if (hub.options.privateRouting) {
- name: string(hub.routing.scriptStorage)
+ name: hub.routing.scriptStorage
dependsOn: [
vNet::scriptSubnet
]
@@ -372,9 +372,11 @@ output config HubProperties = hub
output vNetId string = !hub.options.privateRouting ? '' : vNet.id
@description('Virtual network address prefixes.')
+#disable-next-line BCP318 // Null safety warning for conditional resource access
output vNetAddressSpace array = !hub.options.privateRouting ? [] : vNet.properties.addressSpace.addressPrefixes
@description('Virtual network subnets.')
+#disable-next-line BCP318 // Null safety warning for conditional resource access
output vNetSubnets array = !hub.options.privateRouting ? [] : vNet.properties.subnets
@description('Resource ID of the FinOps hub network subnet.')
diff --git a/src/templates/finops-hub/schemas/settings.json b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/settings.json
similarity index 100%
rename from src/templates/finops-hub/schemas/settings.json
rename to src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/settings.json
diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/RemoteHub/app.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/RemoteHub/app.bicep
new file mode 100644
index 000000000..798cb420d
--- /dev/null
+++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/RemoteHub/app.bicep
@@ -0,0 +1,159 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { finOpsToolkitVersion, HubAppProperties, privateRoutingForLinkedServices } from '../../fx/hub-types.bicep'
+
+
+//==============================================================================
+// Parameters
+//==============================================================================
+
+@description('Required. FinOps hub app getting deployed.')
+param app HubAppProperties
+
+@description('Required. Create and store a key for a remote storage account.')
+@secure()
+param remoteStorageKey string
+
+@description('Required. Remote storage account for ingestion dataset.')
+param remoteHubStorageUri string
+
+@description('Optional. Name of the ingestion container. Default: ingestion.')
+param ingestionContainerName string = 'ingestion'
+
+
+//==============================================================================
+// Variables
+//==============================================================================
+
+var storageKeySecretName = '${toLower(app.hub.name)}-storage-key'
+
+
+//==============================================================================
+// Resources
+//==============================================================================
+
+// App registration
+module appRegistration '../../fx/hub-app.bicep' = {
+ name: 'Microsoft.FinOpsHubs.RemoteHub_Register'
+ params: {
+ app: app
+ version: finOpsToolkitVersion
+ features: [
+ 'DataFactory'
+ 'KeyVault'
+ 'Storage'
+ ]
+ }
+}
+
+// Key Vault secret
+module keyVault_secret '../../fx/hub-vault.bicep' = {
+ name: 'keyVault_secret'
+ params: {
+ vaultName: app.keyVault
+ secretName: storageKeySecretName
+ secretValue: remoteStorageKey
+ secretExpirationInSeconds: 1702648632
+ secretNotBeforeInSeconds: 10000
+ }
+}
+
+// Get key vault instance
+resource keyVault 'Microsoft.KeyVault/vaults@2023-02-01' existing = {
+ name: app.keyVault
+}
+
+// Get data factory instance
+resource dataFactory 'Microsoft.DataFactory/factories@2018-06-01' existing = {
+ name: app.dataFactory
+
+ // cSpell:ignore linkedservices
+ resource linkedService_remoteHubStorage 'linkedservices' = {
+ name: 'remoteHubStorage'
+ properties: {
+ annotations: []
+ parameters: {}
+ type: 'AzureBlobFS'
+ typeProperties: {
+ url: remoteHubStorageUri
+ accountKey: {
+ type: 'AzureKeyVaultSecret'
+ store: {
+ // TODO: Should the key vault linked service name/reference be part of hub settings?
+ referenceName: keyVault.name
+ type: 'LinkedServiceReference'
+ }
+ secretName: storageKeySecretName
+ }
+ }
+ ...privateRoutingForLinkedServices(app.hub)
+ }
+ }
+
+ // Replace the ingestion dataset
+ resource dataset_ingestion 'datasets' = {
+ name: ingestionContainerName
+ properties: {
+ annotations: []
+ parameters: {
+ blobPath: {
+ type: 'String'
+ }
+ }
+ type: 'Parquet'
+ typeProperties: {
+ location: {
+ type: 'AzureBlobFSLocation'
+ fileName: {
+ value: '@{dataset().blobPath}'
+ type: 'Expression'
+ }
+ fileSystem: ingestionContainerName
+ }
+ }
+ linkedServiceName: {
+ parameters: {}
+ referenceName: linkedService_remoteHubStorage.name
+ type: 'LinkedServiceReference'
+ }
+ }
+ }
+
+ // Replace the ingestion_files dataset
+ resource dataset_ingestion_files 'datasets' = {
+ name: '${ingestionContainerName}_files'
+ properties: {
+ annotations: []
+ parameters: {
+ folderPath: {
+ type: 'String'
+ }
+ }
+ type: 'Parquet'
+ typeProperties: {
+ location: {
+ type: 'AzureBlobFSLocation'
+ fileSystem: ingestionContainerName
+ folderPath: {
+ value: '@dataset().folderPath'
+ type: 'Expression'
+ }
+ }
+ }
+ linkedServiceName: {
+ parameters: {}
+ referenceName: linkedService_remoteHubStorage.name
+ type: 'LinkedServiceReference'
+ }
+ }
+ }
+}
+
+
+//==============================================================================
+// Outputs
+//==============================================================================
+
+@description('Name of the Key Vault instance.')
+output keyVaultName string = app.keyVault
diff --git a/src/templates/finops-hub/modules/README.md b/src/templates/finops-hub/modules/README.md
index ba1070efe..f6067314d 100644
--- a/src/templates/finops-hub/modules/README.md
+++ b/src/templates/finops-hub/modules/README.md
@@ -1,13 +1,21 @@
-# 📦 FinOps hub modules
+# 📦 FinOps hub modules and apps
-All FinOps hub module source is available at the root of this directory.
+FinOps hubs consist of reusable `fx` modules and a set of apps, separated by publisher.
-Modules:
+- Publishers define ownership, accountability, and act as a security boundary.
+- Apps should have a single responsibility and provide a complete, comprehensive, and \[generally] self-contained capability.
+- Prefer separate apps to maximize modularity and allow customers to enable or disable holistic features.
+- Apps may rely on functionality from other apps. This is the goal of the extensibility model.
-- [hub.bicep](./hub.bicep) orchestrates the creation of all required resources.
-- [storage.bicep](./storage.bicep) creates the storage account, containers, and settings.json file.
-- [keyVault.bicep](./keyVault.bicep) creates the Key Vault instance and stored secrets.
-- [dataFactory.bicep](./dataFactory.bicep) creates Data Factory pipelines, triggers, etc.
-- [dataExplorer.bicep](./dataExplorer.bicep) creates Data Explorer cluster, database, etc.
+Use the following to help guide decisions about the publisher and app to use for new functionality:
+
+- Who owns and will (or should) maintain the app?
+ - For core FinOps hubs contributors, use `Microsoft.FinOpsHubs`.
+ - For Microsoft product teams, use `Microsoft.{service}` where `{service}` is the owning engineering team.
+ > _NOTE: This includes functionality that would ideally be managed by a service team that is not engaged due to the complexity of the solution. Not every feature should be managed by a separate engineering team. Use your best judgement._
+ - For community-supported features, use `FinOpsToolkit.{area}` where `{area}` is a specific domain or area of responsibility and not a broad customer segment. Community-supported apps will not have the same level of support.
+- Does the functionality share the same security boundary as others (e.g., compliance, data access, permissions)?
+ - If so, use the publisher with the same precise security boundary.
+ - If not, use a new publisher.
diff --git a/src/templates/finops-hub/modules/cm-exports.bicep b/src/templates/finops-hub/modules/cm-exports.bicep
deleted file mode 100644
index e2126a4d8..000000000
--- a/src/templates/finops-hub/modules/cm-exports.bicep
+++ /dev/null
@@ -1,82 +0,0 @@
-// Copyright (c) Microsoft Corporation.
-// Licensed under the MIT License.
-
-import { HubProperties } from 'hub-types.bicep'
-
-
-//==============================================================================
-// Parameters
-//==============================================================================
-
-@description('Required. FinOps hub instance to deploy the app to.')
-param hub HubProperties
-
-
-//==============================================================================
-// Resources
-//==============================================================================
-
-// Register app
-module appRegistration 'hub-app.bicep' = {
- name: 'Microsoft.CostManagement.Exports_Register'
- params: {
- hub: hub
- publisher: 'Microsoft FinOps hubs'
- namespace: 'Microsoft.FinOpsHubs'
- appName: 'Core'
- displayName: 'FinOps hub core'
- appVersion: loadTextContent('ftkver.txt') // cSpell:ignore ftkver
- features: [
- 'DataFactory'
- 'Storage'
- ]
- }
-}
-
-// Upload schema files
-module schemaFiles 'hub-storage.bicep' = {
- name: 'Microsoft.CostManagement.Exports_Storage.SchemaFiles'
- params: {
- app: appRegistration.outputs.app
- container: 'config'
- files: {
- // cSpell:ignore actualcost, amortizedcost, focuscost, pricesheet, reservationdetails, reservationrecommendations, reservationtransactions
- 'schemas/actualcost_c360-2025-04.json': loadTextContent('../schemas/actualcost_c360-2025-04.json')
- 'schemas/amortizedcost_c360-2025-04.json': loadTextContent('../schemas/amortizedcost_c360-2025-04.json')
- 'schemas/focuscost_1.2.json': loadTextContent('../schemas/focuscost_1.2.json')
- 'schemas/focuscost_1.2-preview.json': loadTextContent('../schemas/focuscost_1.2-preview.json')
- 'schemas/focuscost_1.0r2.json': loadTextContent('../schemas/focuscost_1.0r2.json')
- 'schemas/focuscost_1.0.json': loadTextContent('../schemas/focuscost_1.0.json')
- 'schemas/focuscost_1.0-preview(v1).json': loadTextContent('../schemas/focuscost_1.0-preview(v1).json')
- 'schemas/pricesheet_2023-05-01_ea.json': loadTextContent('../schemas/pricesheet_2023-05-01_ea.json')
- 'schemas/pricesheet_2023-05-01_mca.json': loadTextContent('../schemas/pricesheet_2023-05-01_mca.json')
- 'schemas/reservationdetails_2023-03-01.json': loadTextContent('../schemas/reservationdetails_2023-03-01.json')
- 'schemas/reservationrecommendations_2023-05-01_ea.json': loadTextContent('../schemas/reservationrecommendations_2023-05-01_ea.json')
- 'schemas/reservationrecommendations_2023-05-01_mca.json': loadTextContent('../schemas/reservationrecommendations_2023-05-01_mca.json')
- 'schemas/reservationtransactions_2023-05-01_ea.json': loadTextContent('../schemas/reservationtransactions_2023-05-01_ea.json')
- 'schemas/reservationtransactions_2023-05-01_mca.json': loadTextContent('../schemas/reservationtransactions_2023-05-01_mca.json')
- }
- }
-}
-
-// Create msexports container
-module exportContainer 'hub-storage.bicep' = {
- name: 'Microsoft.CostManagement.Exports_Storage.ExportContainer'
- params: {
- app: appRegistration.outputs.app
- container: 'msexports'
- }
-}
-
-// TODO: Add export handling pipelines
-
-
-//==============================================================================
-// Outputs
-//==============================================================================
-
-@description('Name of the container used for Cost Management exports.')
-output exportContainer string = exportContainer.outputs.containerName
-
-@description('Number of schema files uploaded.')
-output schemaFilesUploaded int = schemaFiles.outputs.filesUploaded
diff --git a/src/templates/finops-hub/modules/dataExplorer.bicep b/src/templates/finops-hub/modules/dataExplorer.bicep
deleted file mode 100644
index 86a452e48..000000000
--- a/src/templates/finops-hub/modules/dataExplorer.bicep
+++ /dev/null
@@ -1,501 +0,0 @@
-//==============================================================================
-// Parameters
-//==============================================================================
-
-// @description('Required. Name of the FinOps hub instance. Used to ensure unique resource names.')
-// param hubName string
-
-// @description('Required. Suffix to add to the storage account name to ensure uniqueness.')
-// @minLength(6) // Min length requirement is to avoid a false positive warning
-// param uniqueSuffix string
-
-@description('Optional. Name of the Azure Data Explorer cluster to use for advanced analytics. If empty, Azure Data Explorer will not be deployed. Required to use with Power BI if you have more than $2-5M/mo in costs being monitored. Default: "" (do not use).')
-param clusterName string = ''
-
-// https://learn.microsoft.com/azure/templates/microsoft.kusto/clusters?pivots=deployment-language-bicep#azuresku
-@description('Optional. Name of the Azure Data Explorer SKU. Default: "Dev(No SLA)_Standard_E2a_v4".')
-@allowed([
- 'Dev(No SLA)_Standard_E2a_v4' // 2 CPU, 16GB RAM, 24GB cache, $110/mo
- 'Dev(No SLA)_Standard_D11_v2' // 2 CPU, 14GB RAM, 78GB cache, $121/mo
- 'Standard_D11_v2' // 2 CPU, 14GB RAM, 78GB cache, $245/mo
- 'Standard_D12_v2'
- 'Standard_D13_v2'
- 'Standard_D14_v2'
- 'Standard_D16d_v5'
- 'Standard_D32d_v4'
- 'Standard_D32d_v5'
- 'Standard_DS13_v2+1TB_PS'
- 'Standard_DS13_v2+2TB_PS'
- 'Standard_DS14_v2+3TB_PS'
- 'Standard_DS14_v2+4TB_PS'
- 'Standard_E2a_v4' // 2 CPU, 14GB RAM, 78GB cache, $220/mo
- 'Standard_E2ads_v5'
- 'Standard_E2d_v4'
- 'Standard_E2d_v5'
- 'Standard_E4a_v4'
- 'Standard_E4ads_v5'
- 'Standard_E4d_v4'
- 'Standard_E4d_v5'
- 'Standard_E8a_v4'
- 'Standard_E8ads_v5'
- 'Standard_E8as_v4+1TB_PS'
- 'Standard_E8as_v4+2TB_PS'
- 'Standard_E8as_v5+1TB_PS'
- 'Standard_E8as_v5+2TB_PS'
- 'Standard_E8d_v4'
- 'Standard_E8d_v5'
- 'Standard_E8s_v4+1TB_PS'
- 'Standard_E8s_v4+2TB_PS'
- 'Standard_E8s_v5+1TB_PS'
- 'Standard_E8s_v5+2TB_PS'
- 'Standard_E16a_v4'
- 'Standard_E16ads_v5'
- 'Standard_E16as_v4+3TB_PS'
- 'Standard_E16as_v4+4TB_PS'
- 'Standard_E16as_v5+3TB_PS'
- 'Standard_E16as_v5+4TB_PS'
- 'Standard_E16d_v4'
- 'Standard_E16d_v5'
- 'Standard_E16s_v4+3TB_PS'
- 'Standard_E16s_v4+4TB_PS'
- 'Standard_E16s_v5+3TB_PS'
- 'Standard_E16s_v5+4TB_PS'
- 'Standard_E64i_v3'
- 'Standard_E80ids_v4'
- 'Standard_EC8ads_v5'
- 'Standard_EC8as_v5+1TB_PS'
- 'Standard_EC8as_v5+2TB_PS'
- 'Standard_EC16ads_v5'
- 'Standard_EC16as_v5+3TB_PS'
- 'Standard_EC16as_v5+4TB_PS'
- 'Standard_L4s'
- 'Standard_L8as_v3'
- 'Standard_L8s'
- 'Standard_L8s_v2'
- 'Standard_L8s_v3'
- 'Standard_L16as_v3'
- 'Standard_L16s'
- 'Standard_L16s_v2'
- 'Standard_L16s_v3'
- 'Standard_L32as_v3'
- 'Standard_L32s_v3'
-])
-param clusterSku string = 'Dev(No SLA)_Standard_E2a_v4'
-
-@description('Optional. Number of nodes to use in the cluster. Allowed values: 1 for the Basic SKU tier and 2-1000 for Standard. Default: 1 for dev/test SKUs, 2 for standard SKUs.')
-@minValue(1)
-@maxValue(1000)
-param clusterCapacity int = 1
-
-// TODO: Figure out why this is breaking upgrades
-// @description('Optional. Array of external tenant IDs that should have access to the cluster. Default: empty (no external access).')
-// param clusterTrustedExternalTenants string[] = []
-
-@description('Optional. Forces the table to be updated if different from the last time it was deployed.')
-param forceUpdateTag string = utcNow()
-
-@description('Optional. If true, ingestion will continue even if some rows fail to ingest. Default: false.')
-param continueOnErrors bool = false
-
-@description('Optional. Azure location to use for the managed identity and deployment script to auto-start triggers. Default: (resource group location).')
-param location string = resourceGroup().location
-
-@description('Optional. Tags to apply to all resources.')
-param tags object = {}
-
-@description('Optional. Tags to apply to resources based on their resource type. Resource type specific tags will be merged with tags for all resources.')
-param tagsByResource object = {}
-
-@description('Required. Name of the Data Factory instance.')
-param dataFactoryName string
-
-@description('Optional. Number of days of data to retain in the Data Explorer *_raw tables. Default: 0.')
-param rawRetentionInDays int = 0
-
-@description('Required. Name of the storage account to use for data ingestion.')
-param storageAccountName string
-
-@description('Required. Resource ID of the virtual network for private endpoints.')
-param virtualNetworkId string
-
-@description('Required. Resource ID of the subnet for private endpoints.')
-param privateEndpointSubnetId string
-
-@description('Optional. Enable public access.')
-param enablePublicAccess bool
-
-//------------------------------------------------------------------------------
-// Variables
-//------------------------------------------------------------------------------
-
-// cSpell:ignore ftkver, privatelink
-var dataExplorerPrivateDnsZoneName = replace('privatelink.${location}.${replace(environment().suffixes.storage, 'core', 'kusto')}', '..', '.')
-
-// Actual = Minimum(ClusterMaximumConcurrentOperations, Number of nodes in cluster * Maximum(1, Core count per node * CoreUtilizationCoefficient))
-var ingestionCapacity = {
- 'Dev(No SLA)_Standard_E2a_v4': 1
- 'Dev(No SLA)_Standard_D11_v2': 1
- Standard_D11_v2: 2
- Standard_D12_v2: 4
- Standard_D13_v2: 8
- Standard_D14_v2: 16
- Standard_D16d_v5: 16
- Standard_D32d_v4: 32
- Standard_D32d_v5: 32
- 'Standard_DS13_v2+1TB_PS': 8
- 'Standard_DS13_v2+2TB_PS': 8
- 'Standard_DS14_v2+3TB_PS': 16
- 'Standard_DS14_v2+4TB_PS': 16
- Standard_E2a_v4: 2
- Standard_E2ads_v5: 2
- Standard_E2d_v4: 2
- Standard_E2d_v5: 2
- Standard_E4a_v4: 4
- Standard_E4ads_v5: 4
- Standard_E4d_v4: 4
- Standard_E4d_v5: 4
- Standard_E8a_v4: 8
- Standard_E8ads_v5: 8
- 'Standard_E8as_v4+1TB_PS': 8
- 'Standard_E8as_v4+2TB_PS': 8
- 'Standard_E8as_v5+1TB_PS': 8
- 'Standard_E8as_v5+2TB_PS': 8
- Standard_E8d_v4: 8
- Standard_E8d_v5: 8
- 'Standard_E8s_v4+1TB_PS': 8
- 'Standard_E8s_v4+2TB_PS': 8
- 'Standard_E8s_v5+1TB_PS': 8
- 'Standard_E8s_v5+2TB_PS': 8
- Standard_E16a_v4: 16
- Standard_E16ads_v5: 16
- 'Standard_E16as_v4+3TB_PS': 16
- 'Standard_E16as_v4+4TB_PS': 16
- 'Standard_E16as_v5+3TB_PS': 16
- 'Standard_E16as_v5+4TB_PS': 16
- Standard_E16d_v4: 16
- Standard_E16d_v5: 16
- 'Standard_E16s_v4+3TB_PS': 16
- 'Standard_E16s_v4+4TB_PS': 16
- 'Standard_E16s_v5+3TB_PS': 16
- 'Standard_E16s_v5+4TB_PS': 16
- Standard_E64i_v3: 64
- Standard_E80ids_v4: 80
- Standard_EC8ads_v5: 8
- 'Standard_EC8as_v5+1TB_PS': 8
- 'Standard_EC8as_v5+2TB_PS': 8
- Standard_EC16ads_v5: 16
- 'Standard_EC16as_v5+3TB_PS': 16
- 'Standard_EC16as_v5+4TB_PS': 16
- Standard_L4s: 4
- Standard_L8as_v3: 8
- Standard_L8s: 8
- Standard_L8s_v2: 8
- Standard_L8s_v3: 8
- Standard_L16as_v3: 16
- Standard_L16s: 16
- Standard_L16s_v2: 16
- Standard_L16s_v3: 16
- Standard_L32as_v3: 32
- Standard_L32s_v3: 32
-}
-
-//==============================================================================
-// Resources
-//==============================================================================
-
-//------------------------------------------------------------------------------
-// Dependencies
-//------------------------------------------------------------------------------
-
-// Get data factory instance
-resource dataFactory 'Microsoft.DataFactory/factories@2018-06-01' existing = {
- name: dataFactoryName
-}
-
-resource blobPrivateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' existing = {
- name: 'privatelink.blob.${environment().suffixes.storage}'
-}
-
-resource queuePrivateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' existing = {
- name: 'privatelink.queue.${environment().suffixes.storage}'
-}
-
-resource tablePrivateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' existing = {
- name: 'privatelink.table.${environment().suffixes.storage}'
-}
-
-resource storage 'Microsoft.Storage/storageAccounts@2022-09-01' existing = {
- name: storageAccountName
-}
-
-//------------------------------------------------------------------------------
-// Cluster + databases
-//------------------------------------------------------------------------------
-
-// Kusto cluster
-resource cluster 'Microsoft.Kusto/clusters@2023-08-15' = {
- name: clusterName
- location: location
- tags: union(tags, tagsByResource[?'Microsoft.Kusto/clusters'] ?? {})
- sku: {
- name: clusterSku
- tier: startsWith(clusterSku, 'Dev(No SLA)_') ? 'Basic' : 'Standard'
- capacity: startsWith(clusterSku, 'Dev(No SLA)_') ? 1 : (clusterCapacity == 1 ? 2 : clusterCapacity)
- }
- identity: {
- type: 'SystemAssigned'
- }
- properties: {
- enableStreamingIngest: true
- enableAutoStop: false
- publicNetworkAccess: enablePublicAccess ? 'Enabled' : 'Disabled'
- // TODO: Figure out why this is breaking upgrades
- // trustedExternalTenants: [for tenantId in clusterTrustedExternalTenants: {
- // value: tenantId
- // }]
- }
-
- resource adfClusterAdmin 'principalAssignments' = {
- name: 'adf-mi-cluster-admin'
- properties: {
- principalType: 'App'
- principalId: dataFactory.identity.principalId
- tenantId: dataFactory.identity.tenantId
- role: 'AllDatabasesAdmin'
- }
- }
-
- resource ingestionDb 'databases' = {
- name: 'Ingestion'
- location: location
- kind: 'ReadWrite'
- }
-
- resource hubDb 'databases' = {
- name: 'Hub'
- location: location
- kind: 'ReadWrite'
- }
-}
-
-module ingestion_OpenDataInternalScripts 'hub-database.bicep' = {
- name: 'ingestion_OpenDataInternalScripts'
- params: {
- clusterName: cluster.name
- databaseName: cluster::ingestionDb.name
- scripts: {
- OpenDataFunctions_resource_type_1: loadTextContent('scripts/OpenDataFunctions_resource_type_1.kql')
- OpenDataFunctions_resource_type_2: loadTextContent('scripts/OpenDataFunctions_resource_type_2.kql')
- OpenDataFunctions_resource_type_3: loadTextContent('scripts/OpenDataFunctions_resource_type_3.kql')
- OpenDataFunctions_resource_type_4: loadTextContent('scripts/OpenDataFunctions_resource_type_4.kql')
- OpenDataFunctions_resource_type_5: loadTextContent('scripts/OpenDataFunctions_resource_type_5.kql')
- }
- continueOnErrors: continueOnErrors
- forceUpdateTag: forceUpdateTag
- }
-}
-
-module ingestion_InitScripts 'hub-database.bicep' = {
- name: 'ingestion_InitScripts'
- dependsOn: [
- ingestion_OpenDataInternalScripts
- ]
- params: {
- clusterName: cluster.name
- databaseName: cluster::ingestionDb.name
- scripts: {
- openData: loadTextContent('scripts/OpenDataFunctions.kql')
- common: loadTextContent('scripts/Common.kql')
- infra: loadTextContent('scripts/IngestionSetup_HubInfra.kql')
- rawTables: replace(loadTextContent('scripts/IngestionSetup_RawTables.kql'), '$$rawRetentionInDays$$', string(rawRetentionInDays))
- }
- continueOnErrors: continueOnErrors
- forceUpdateTag: forceUpdateTag
- }
-}
-
-module ingestion_VersionedScripts 'hub-database.bicep' = {
- name: 'ingestion_VersionedScripts'
- dependsOn: [
- ingestion_InitScripts
- ]
- params: {
- clusterName: cluster.name
- databaseName: cluster::ingestionDb.name
- scripts: {
- v1_0: loadTextContent('scripts/IngestionSetup_v1_0.kql')
- v1_2: loadTextContent('scripts/IngestionSetup_v1_2.kql')
- }
- continueOnErrors: continueOnErrors
- forceUpdateTag: forceUpdateTag
- }
-}
-
-module hub_InitScripts 'hub-database.bicep' = {
- name: 'hub_InitScripts'
- dependsOn: [
- ingestion_InitScripts
- ]
- params: {
- clusterName: cluster.name
- databaseName: cluster::hubDb.name
- scripts: {
- common: loadTextContent('scripts/Common.kql')
- openData: loadTextContent('scripts/HubSetup_OpenData.kql')
- }
- continueOnErrors: continueOnErrors
- forceUpdateTag: forceUpdateTag
- }
-}
-
-module hub_VersionedScripts 'hub-database.bicep' = {
- name: 'hub_VersionedScripts'
- dependsOn: [
- ingestion_VersionedScripts
- hub_InitScripts
- ]
- params: {
- clusterName: cluster.name
- databaseName: cluster::hubDb.name
- scripts: {
- v1_0: loadTextContent('scripts/HubSetup_v1_0.kql')
- v1_2: loadTextContent('scripts/HubSetup_v1_2.kql')
- }
- continueOnErrors: continueOnErrors
- forceUpdateTag: forceUpdateTag
- }
-}
-
-module hub_LatestScripts 'hub-database.bicep' = {
- name: 'hub_LatestScripts'
- dependsOn: [
- hub_VersionedScripts
- ]
- params: {
- clusterName: cluster.name
- databaseName: cluster::hubDb.name
- scripts: {
- latest: loadTextContent('scripts/HubSetup_Latest.kql')
- }
- continueOnErrors: continueOnErrors
- forceUpdateTag: forceUpdateTag
- }
-}
-
-// Authorize Kusto Cluster to read storage
-resource clusterStorageAccess 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
- name: guid(cluster.name, subscription().id, 'Storage Blob Data Contributor')
- scope: storage
- properties: {
- description: 'Give "Storage Blob Data Contributor" to the cluster'
- principalId: cluster.identity.principalId
- // Required in case principal not ready when deploying the assignment
- principalType: 'ServicePrincipal'
- roleDefinitionId: subscriptionResourceId(
- 'Microsoft.Authorization/roleDefinitions',
- 'ba92f5b4-2d11-453d-a403-e96b0029c9fe' // Storage Blob Data Contributor -- https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#storage
- )
- }
-}
-
-// DNS zone
-resource dataExplorerPrivateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' = if (!enablePublicAccess) {
- name: dataExplorerPrivateDnsZoneName
- location: 'global'
- tags: union(tags, tagsByResource[?'Microsoft.Network/privateDnsZones'] ?? {})
- properties: {}
-}
-
-// Link DNS zone to VNet
-resource dataExplorerPrivateDnsZoneLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2024-06-01' = if (!enablePublicAccess) {
- name: '${replace(dataExplorerPrivateDnsZone.name, '.', '-')}-link'
- location: 'global'
- parent: dataExplorerPrivateDnsZone
- tags: union(tags, tagsByResource[?'Microsoft.Network/privateDnsZones/virtualNetworkLinks'] ?? {})
- properties: {
- virtualNetwork: {
- id: virtualNetworkId
- }
- registrationEnabled: false
- }
-}
-
-// Private endpoint
-resource dataExplorerEndpoint 'Microsoft.Network/privateEndpoints@2023-11-01' = if (!enablePublicAccess) {
- name: '${cluster.name}-ep'
- location: location
- tags: union(tags, tagsByResource[?'Microsoft.Network/privateEndpoints'] ?? {})
- properties: {
- subnet: {
- id: privateEndpointSubnetId
- }
- privateLinkServiceConnections: [
- {
- name: 'dataExplorerLink'
- properties: {
- privateLinkServiceId: cluster.id
- groupIds: ['cluster']
- }
- }
- ]
- }
-}
-
-// DNS records for private endpoint
-resource dataExplorerPrivateDnsZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2023-11-01' = if (!enablePublicAccess) {
- name: 'dataExplorer-endpoint-zone'
- parent: dataExplorerEndpoint
- properties: {
- privateDnsZoneConfigs: [
- {
- name: 'privatelink-westus-kusto-net'
- properties: {
- privateDnsZoneId: dataExplorerPrivateDnsZone.id
- }
- }
- {
- name: 'privatelink-blob-core-windows-net'
- properties: {
- privateDnsZoneId: blobPrivateDnsZone.id
- }
- }
- {
- name: 'privatelink-table-core-windows-net'
- properties: {
- privateDnsZoneId: tablePrivateDnsZone.id
- }
- }
- {
- name: 'privatelink-queue-core-windows-net'
- properties: {
- privateDnsZoneId: queuePrivateDnsZone.id
- }
- }
- ]
- }
-}
-
-//==============================================================================
-// Outputs
-//==============================================================================
-
-@description('The resource ID of the cluster.')
-output clusterId string = cluster.id
-
-@description('The ID of the cluster system assigned managed identity.')
-output principalId string = cluster.identity.principalId
-
-@description('The name of the cluster.')
-output clusterName string = cluster.name
-
-@description('The URI of the cluster.')
-output clusterUri string = cluster.properties.uri
-
-@description('The name of the database for data ingestion.')
-output ingestionDbName string = cluster::ingestionDb.name
-
-@description('The name of the database for queries.')
-output hubDbName string = cluster::hubDb.name
-
-@description('Max ingestion capacity of the cluster.')
-output clusterIngestionCapacity int = ingestionCapacity[?clusterSku] ?? 1
diff --git a/src/templates/finops-hub/modules/dataFactory.bicep b/src/templates/finops-hub/modules/dataFactory.bicep
deleted file mode 100644
index 2c7fe53e5..000000000
--- a/src/templates/finops-hub/modules/dataFactory.bicep
+++ /dev/null
@@ -1,5251 +0,0 @@
-// Copyright (c) Microsoft Corporation.
-// Licensed under the MIT License.
-
-import { HubAppProperties } from 'hub-types.bicep'
-
-
-//==============================================================================
-// Parameters
-//==============================================================================
-
-@description('Required. Temporary app placeholder for the deployments module.')
-param app HubAppProperties
-
-@description('Required. Name of the FinOps hub instance.')
-param hubName string
-
-@description('Required. Name of the Data Factory instance.')
-param dataFactoryName string
-
-@description('Required. The name of the Azure Key Vault instance.')
-param keyVaultName string
-
-@description('Required. The name of the Azure storage account instance.')
-param storageAccountName string
-
-@description('Required. The name of the container where Cost Management data is exported.')
-param exportContainerName string
-
-@description('Required. The name of the container where normalized data is ingested.')
-param ingestionContainerName string
-
-@description('Required. The name of the container where normalized data is ingested.')
-param configContainerName string
-
-@description('Optional. Name of the Azure Data Explorer cluster to use for advanced analytics, if applicable.')
-param dataExplorerName string = ''
-
-@description('Optional. Resource ID of the Azure Data Explorer cluster to use for advanced analytics, if applicable.')
-param dataExplorerId string = ''
-
-@description('Optional. ID of the Azure Data Explorer cluster system assigned managed identity, if applicable.')
-param dataExplorerPrincipalId string = ''
-
-// cSpell:ignore eventhouse
-@description('Optional. URI of the Azure Data Explorer cluster or Microsoft Fabric eventhouse query endpoint to use for advanced analytics, if applicable.')
-param dataExplorerUri string = ''
-
-@description('Optional. Name of the Azure Data Explorer ingestion database. Default: "ingestion".')
-param dataExplorerIngestionDatabase string = 'Ingestion'
-
-@description('Optional. Azure Data Explorer ingestion capacity or Microsoft Fabric capacity units. Increase for non-dev/trial SKUs. Default: 1')
-param dataExplorerIngestionCapacity int = 1
-
-@description('Optional. The location to use for the managed identity and deployment script to auto-start triggers. Default = (resource group location).')
-param location string = resourceGroup().location
-
-@description('Optional. Remote storage account for ingestion dataset.')
-param remoteHubStorageUri string
-
-@description('Optional. Tags to apply to all resources.')
-param tags object = {}
-
-@description('Optional. Tags to apply to resources based on their resource type. Resource type specific tags will be merged with tags for all resources.')
-param tagsByResource object = {}
-
-@description('Optional. Enable managed exports where your FinOps hub instance will create and run Cost Management exports on your behalf. Not supported for Microsoft Customer Agreement (MCA) billing profiles. Requires the ability to grant User Access Administrator role to FinOps hubs, which is required to create Cost Management exports. Default: true.')
-param enableManagedExports bool = true
-
-@description('Required. Enable public access.')
-param enablePublicAccess bool
-
-//------------------------------------------------------------------------------
-// Variables
-//------------------------------------------------------------------------------
-
-var focusSchemaVersion = '1.0'
-var exportSchemaVersion = '2023-05-01'
-var reservationDetailsSchemaVersion = '2023-03-01'
-// cSpell:ignore ftkver
-var ftkVersion = loadTextContent('ftkver.txt')
-var ftkReleaseUri = endsWith(ftkVersion, '-dev')
- ? 'https://github.com/microsoft/finops-toolkit/releases/latest/download'
- : 'https://github.com/microsoft/finops-toolkit/releases/download/v${ftkVersion}'
-var exportApiVersion = '2023-07-01-preview'
-var hubDataExplorerName = 'hubDataExplorer'
-
-// cSpell:ignore timeframe
-// Function to generate the body for a Cost Management export
-func getExportBody(exportContainerName string, datasetType string, schemaVersion string, isMonthly bool, exportFormat string, compressionMode string, partitionData string, dataOverwriteBehavior string) string => '{ "properties": { "definition": { "dataSet": { "configuration": { "dataVersion": "${schemaVersion}", "filters": [] }, "granularity": "Daily" }, "timeframe": "${isMonthly ? 'TheLastMonth': 'MonthToDate' }", "type": "${datasetType}" }, "deliveryInfo": { "destination": { "container": "${exportContainerName}", "rootFolderPath": "@{if(startswith(item().scope, \'/\'), substring(item().scope, 1, sub(length(item().scope), 1)) ,item().scope)}", "type": "AzureBlob", "resourceId": "@{variables(\'storageAccountId\')}" } }, "schedule": { "recurrence": "${ isMonthly ? 'Monthly' : 'Daily'}", "recurrencePeriod": { "from": "2024-01-01T00:00:00.000Z", "to": "2050-02-01T00:00:00.000Z" }, "status": "Inactive" }, "format": "${exportFormat}", "partitionData": "${partitionData}", "dataOverwriteBehavior": "${dataOverwriteBehavior}", "compressionMode": "${compressionMode}" }, "id": "@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{variables(\'exportName\')}", "name": "@{variables(\'exportName\')}", "type": "Microsoft.CostManagement/reports", "identity": { "type": "systemAssigned" }, "location": "global" }'
-
-func getExportBodyV2(exportContainerName string, datasetType string, schemaVersion string, isMonthly bool, exportFormat string, compressionMode string, partitionData string, dataOverwriteBehavior string, recommendationScope string, recommendationLookbackPeriod string, resourceType string) string => /*
- */ toLower(datasetType) == 'focuscost' ? /*
- */ '{ "properties": { "definition": { "dataSet": { "configuration": { "dataVersion": "${schemaVersion}", "filters": [] }, "granularity": "Daily" }, "timeframe": "${isMonthly ? 'TheLastMonth': 'MonthToDate' }", "type": "${datasetType}" }, "deliveryInfo": { "destination": { "container": "${exportContainerName}", "rootFolderPath": "@{if(startswith(item().scope, \'/\'), substring(item().scope, 1, sub(length(item().scope), 1)) ,item().scope)}", "type": "AzureBlob", "resourceId": "@{variables(\'storageAccountId\')}" } }, "schedule": { "recurrence": "${ isMonthly ? 'Monthly' : 'Daily'}", "recurrencePeriod": { "from": "2024-01-01T00:00:00.000Z", "to": "2050-02-01T00:00:00.000Z" }, "status": "Inactive" }, "format": "${exportFormat}", "partitionData": "${partitionData}", "dataOverwriteBehavior": "${dataOverwriteBehavior}", "compressionMode": "${compressionMode}" }, "id": "@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{toLower(concat(variables(\'finOpsHub\'), \'-${ isMonthly ? 'monthly' : 'daily'}-costdetails\'))}", "name": "@{toLower(concat(variables(\'finOpsHub\'), \'-${ isMonthly ? 'monthly' : 'daily'}-costdetails\'))}", "type": "Microsoft.CostManagement/reports", "identity": { "type": "systemAssigned" }, "location": "global" }' /*
- */ : toLower(datasetType) == 'reservationdetails' ? /*
- */ '{ "properties": { "definition": { "dataSet": { "configuration": { "dataVersion": "${schemaVersion}", "filters": [] }, "granularity": "Daily" }, "timeframe": "${isMonthly ? 'TheLastMonth': 'MonthToDate' }", "type": "${datasetType}" }, "deliveryInfo": { "destination": { "container": "${exportContainerName}", "rootFolderPath": "@{if(startswith(item().scope, \'/\'), substring(item().scope, 1, sub(length(item().scope), 1)) ,item().scope)}", "type": "AzureBlob", "resourceId": "@{variables(\'storageAccountId\')}" } }, "schedule": { "recurrence": "${ isMonthly ? 'Monthly' : 'Daily'}", "recurrencePeriod": { "from": "2024-01-01T00:00:00.000Z", "to": "2050-02-01T00:00:00.000Z" }, "status": "Inactive" }, "format": "${exportFormat}", "partitionData": "${partitionData}", "dataOverwriteBehavior": "${dataOverwriteBehavior}", "compressionMode": "${compressionMode}" }, "id": "@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{toLower(concat(variables(\'finOpsHub\'), \'-${ isMonthly ? 'monthly' : 'daily'}-${toLower(datasetType)}\'))}", "name": "@{toLower(concat(variables(\'finOpsHub\'), \'-${ isMonthly ? 'monthly' : 'daily'}-${toLower(datasetType)}\'))}", "type": "Microsoft.CostManagement/reports", "identity": { "type": "systemAssigned" }, "location": "global" }' /*
- */ : (toLower(datasetType) == 'pricesheet') || (toLower(datasetType) == 'reservationtransactions') ? /*
- */ '{ "properties": { "definition": { "dataSet": { "configuration": { "dataVersion": "${schemaVersion}", "filters": [] }}, "timeframe": "${isMonthly ? 'TheCurrentMonth': 'MonthToDate' }", "type": "${datasetType}" }, "deliveryInfo": { "destination": { "container": "${exportContainerName}", "rootFolderPath": "@{if(startswith(item().scope, \'/\'), substring(item().scope, 1, sub(length(item().scope), 1)) ,item().scope)}", "type": "AzureBlob", "resourceId": "@{variables(\'storageAccountId\')}" } }, "schedule": { "recurrence": "${ isMonthly ? 'Monthly' : 'Daily'}", "recurrencePeriod": { "from": "2024-01-01T00:00:00.000Z", "to": "2050-02-01T00:00:00.000Z" }, "status": "Inactive" }, "format": "${exportFormat}", "partitionData": "${partitionData}", "dataOverwriteBehavior": "${dataOverwriteBehavior}", "compressionMode": "${compressionMode}" }, "id": "@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{toLower(concat(variables(\'finOpsHub\'), \'-${ isMonthly ? 'monthly' : 'daily'}-${toLower(datasetType)}\'))}", "name": "@{toLower(concat(variables(\'finOpsHub\'), \'-${ isMonthly ? 'monthly' : 'daily'}-${toLower(datasetType)}\'))}", "type": "Microsoft.CostManagement/reports", "identity": { "type": "systemAssigned" }, "location": "global" }' /*
- */ : toLower(datasetType) == 'reservationrecommendations' ? /*
- */ '{ "properties": { "definition": { "dataSet": { "configuration": { "dataVersion": "${schemaVersion}", "filters": [ { "name": "reservationScope", "value": "${recommendationScope}" }, { "name": "resourceType", "value": "${resourceType}" }, { "name": "lookBackPeriod", "value": "${recommendationLookbackPeriod}" }] }}, "timeframe": "${isMonthly ? 'TheLastMonth': 'MonthToDate' }", "type": "${datasetType}" }, "deliveryInfo": { "destination": { "container": "${exportContainerName}", "rootFolderPath": "@{if(startswith(item().scope, \'/\'), substring(item().scope, 1, sub(length(item().scope), 1)) ,item().scope)}", "type": "AzureBlob", "resourceId": "@{variables(\'storageAccountId\')}" } }, "schedule": { "recurrence": "${ isMonthly ? 'Monthly' : 'Daily'}", "recurrencePeriod": { "from": "2024-01-01T00:00:00.000Z", "to": "2050-02-01T00:00:00.000Z" }, "status": "Inactive" }, "format": "${exportFormat}", "partitionData": "${partitionData}", "dataOverwriteBehavior": "${dataOverwriteBehavior}", "compressionMode": "${compressionMode}" }, "id": "@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{toLower(concat(variables(\'finOpsHub\'), \'-${ isMonthly ? 'monthly' : 'daily'}-costdetails\'))}", "name": "@{toLower(concat(variables(\'finOpsHub\'), \'-${ isMonthly ? 'monthly' : 'daily'}-costdetails\'))}", "type": "Microsoft.CostManagement/reports", "identity": { "type": "systemAssigned" }, "location": "global" }' /*
- */ : 'undefined'
-
-var deployDataExplorer = !empty(dataExplorerId)
-var useFabric = !deployDataExplorer && !empty(dataExplorerUri)
-
-var datasetPropsDefault = {
- location: {
- type: 'AzureBlobFSLocation'
- fileName: {
- value: '@{dataset().fileName}'
- type: 'Expression'
- }
- folderPath: {
- value: '@{dataset().folderPath}'
- type: 'Expression'
- }
- }
-}
-
-var safeExportContainerName = replace('${exportContainerName}', '-', '_')
-var safeIngestionContainerName = replace('${ingestionContainerName}', '-', '_')
-var safeConfigContainerName = replace('${configContainerName}', '-', '_')
-// cSpell:ignore vnet
-var managedVnetName = 'default'
-
-// Separator used to separate ingestion ID from file name for ingested files
-var ingestionIdFileNameSeparator = '__'
-
-// All hub triggers (used to auto-start)
-var exportManifestAddedTriggerName = '${safeExportContainerName}_ManifestAdded'
-var ingestionManifestAddedTriggerName = '${safeIngestionContainerName}_ManifestAdded'
-var updateConfigTriggerName = '${safeConfigContainerName}_SettingsUpdated'
-var dailyTriggerName = '${safeConfigContainerName}_DailySchedule'
-var monthlyTriggerName = '${safeConfigContainerName}_MonthlySchedule'
-var allHubTriggers = [
- exportManifestAddedTriggerName
- ingestionManifestAddedTriggerName
- updateConfigTriggerName
- dailyTriggerName
- monthlyTriggerName
-]
-
-// Roles needed to auto-start triggers
-var autoStartRbacRoles = [
- // Data Factory contributor -- https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#data-factory-contributor
- // Used to start/stop triggers and delete old pipelines/triggers
- '673868aa-7521-48a0-acc6-0f60742d39f5'
-]
-
-// Roles for ADF to manage data in storage
-// Does not include roles assignments needed against the export scope
-var storageRbacRoles = union (
- [
- // Storage Account Contributor -- https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#storage-account-contributor
- // Used to move files from the msexports to ingestion container
- '17d1049b-9a84-46fb-8f53-869881c3d3ab'
- // Storage Blob Data Contributor -- https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#storage-blob-data-contributor
- 'ba92f5b4-2d11-453d-a403-e96b0029c9fe'
- // Reader -- https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#reader
- 'acdd72a7-3385-48ef-bd42-f606fba81ae7'
- ],
- // Only use User Access Administrator if managed exports are enabled for least privileged access
- !enableManagedExports ? [] : [
- // User Access Administrator -- https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#user-access-administrator
- '18d7d88d-d35e-4fb5-a5c3-7773c20a72d9'
- ]
-)
-
-// Roles for ADF to to start check ADX cluster and to start cluster if stopped
-var adxRbacRoles = [
- 'b24988ac-6180-42a0-ab88-20f7382dd24c' // Contributor permissions on the cluster
-]
-
-//==============================================================================
-// Resources
-//==============================================================================
-
-// Get data factory instance
-resource dataFactory 'Microsoft.DataFactory/factories@2018-06-01' existing = {
- name: dataFactoryName
-}
-
-// Get storage account instance
-resource storageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' existing = {
- name: storageAccountName
-}
-
-// Get keyvault instance
-resource keyVault 'Microsoft.KeyVault/vaults@2023-02-01' existing = if (!empty(remoteHubStorageUri)) {
- name: keyVaultName
-}
-
-// Get ADX cluster instance
-resource dataExplorerCluster 'Microsoft.Kusto/clusters@2023-08-15' existing = if (deployDataExplorer) {
- name: dataExplorerName
-}
-
-// cSpell:ignore azuretimezones
-module azuretimezones 'azuretimezones.bicep' = {
- name: 'azuretimezones'
- params: {
- location: location
- }
-}
-
-resource managedVirtualNetwork 'Microsoft.DataFactory/factories/managedVirtualNetworks@2018-06-01' = if (!enablePublicAccess) {
- name: managedVnetName
- parent: dataFactory
- properties: {}
-}
-
-resource managedIntegrationRuntime 'Microsoft.DataFactory/factories/integrationRuntimes@2018-06-01' = if (!enablePublicAccess) {
- name: 'ManagedIntegrationRuntime'
- parent: dataFactory
- properties: {
- type: 'Managed'
- managedVirtualNetwork: {
- referenceName: managedVnetName
- type: 'ManagedVirtualNetworkReference'
- }
- typeProperties: {
- computeProperties: {
- location: location
- dataFlowProperties: {
- computeType: 'General'
- coreCount: 8
- timeToLive: 10
- cleanup: false
- customProperties: []
- }
- copyComputeScaleProperties: {
- dataIntegrationUnit: 16
- timeToLive: 30
- }
- pipelineExternalComputeScaleProperties: {
- timeToLive: 30
- numberOfPipelineNodes: 1
- numberOfExternalNodes: 1
- }
- }
- }
- }
- dependsOn: [
- managedVirtualNetwork
- ]
-}
-
-resource storageManagedPrivateEndpoint 'Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints@2018-06-01' = if (!enablePublicAccess) {
- name: storageAccount.name
- parent: managedVirtualNetwork
- properties: {
- name: storageAccount.name
- groupId: 'dfs'
- privateLinkResourceId: storageAccount.id
- fqdns: [
- storageAccount.properties.primaryEndpoints.dfs
- ]
- }
-}
-
-module getStoragePrivateEndpointConnections 'storageEndpoints.bicep' = if (!enablePublicAccess) {
- name: 'GetStoragePrivateEndpointConnections'
- dependsOn: [
- storageManagedPrivateEndpoint
- ]
- params: {
- storageAccountName: storageAccount.name
- }
-}
-
-module approveStoragePrivateEndpointConnections 'storageEndpoints.bicep' = if (!enablePublicAccess) {
- name: 'ApproveStoragePrivateEndpointConnections'
- params: {
- storageAccountName: storageAccount.name
- privateEndpointConnections: getStoragePrivateEndpointConnections.outputs.privateEndpointConnections
- }
-}
-
-resource keyVaultManagedPrivateEndpoint 'Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints@2018-06-01' = if (!empty(remoteHubStorageUri) && !enablePublicAccess) {
- name: keyVault.name
- parent: managedVirtualNetwork
- properties: {
- name: keyVault.name
- groupId: 'vault'
- privateLinkResourceId: keyVault.id
- fqdns: [
- keyVault.properties.vaultUri
- ]
- }
-}
-
-module getKeyVaultPrivateEndpointConnections 'keyVaultEndpoints.bicep' = if (!empty(remoteHubStorageUri) && !enablePublicAccess) {
- name: 'GetKeyVaultPrivateEndpointConnections'
- dependsOn: [
- keyVaultManagedPrivateEndpoint
- ]
- params: {
- keyVaultName: keyVault.name
- }
-}
-
-module approveKeyVaultPrivateEndpointConnections 'keyVaultEndpoints.bicep' = if (!empty(remoteHubStorageUri) && !enablePublicAccess) {
- name: 'ApproveKeyVaultPrivateEndpointConnections'
- params: {
- keyVaultName: keyVault.name
- privateEndpointConnections: getKeyVaultPrivateEndpointConnections.outputs.privateEndpointConnections
- }
-}
-
-resource dataExplorerManagedPrivateEndpoint 'Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints@2018-06-01' = if (deployDataExplorer && !enablePublicAccess) {
- name: hubDataExplorerName
- parent: managedVirtualNetwork
- properties: {
- name: hubDataExplorerName
- groupId: 'cluster'
- privateLinkResourceId: dataExplorerId
- fqdns: [
- dataExplorerUri
- ]
- }
-}
-
-module getDataExplorerPrivateEndpointConnections 'dataExplorerEndpoints.bicep' = if (deployDataExplorer && !enablePublicAccess) {
- name: 'GetDataExplorerPrivateEndpointConnections'
- dependsOn: [
- dataExplorerManagedPrivateEndpoint
- ]
- params: {
- dataExplorerName: dataExplorerName
- }
-}
-
-module approveDataExplorerPrivateEndpointConnections 'dataExplorerEndpoints.bicep' = if (deployDataExplorer && !enablePublicAccess) {
- name: 'ApproveDataExplorerPrivateEndpointConnections'
- params: {
- dataExplorerName: dataExplorerName
- privateEndpointConnections: getDataExplorerPrivateEndpointConnections.outputs.privateEndpointConnections
- }
-}
-
-//------------------------------------------------------------------------------
-// Identities and RBAC
-//------------------------------------------------------------------------------
-
-// Create managed identity to start/stop triggers
-resource triggerManagerIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = {
- name: '${dataFactory.name}_triggerManager'
- location: location
- tags: union(tags, tagsByResource[?'Microsoft.ManagedIdentity/userAssignedIdentities'] ?? {})
-}
-
-resource triggerManagerRoleAssignments 'Microsoft.Authorization/roleAssignments@2022-04-01' = [for role in autoStartRbacRoles: {
- name: guid(dataFactory.id, role, triggerManagerIdentity.id)
- scope: dataFactory
- properties: {
- roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', role)
- principalId: triggerManagerIdentity.properties.principalId
- principalType: 'ServicePrincipal'
- }
-}]
-
-// Grant ADF identity access to manage data in storage
-resource factoryIdentityStorageRoleAssignments 'Microsoft.Authorization/roleAssignments@2022-04-01' = [for role in storageRbacRoles: {
- name: guid(storageAccount.id, role, dataFactory.id)
- scope: storageAccount
- properties: {
- roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', role)
- principalId: dataFactory.identity.principalId
- principalType: 'ServicePrincipal'
- }
-}]
-
-// Grant ADF identity access to manage ADX cluster
-resource factoryIdentityDataExplorerRoleAssignments 'Microsoft.Authorization/roleAssignments@2022-04-01' = [for role in adxRbacRoles: if (deployDataExplorer) {
- name: guid(dataExplorerCluster.id, role, dataFactory.id)
- scope: dataExplorerCluster
- properties: {
- roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', role)
- principalId: dataFactory.identity.principalId
- principalType: 'ServicePrincipal'
- }
-}]
-
-//------------------------------------------------------------------------------
-// Delete old triggers and pipelines
-//------------------------------------------------------------------------------
-
-module deleteOldResources 'hub-deploymentScript.bicep' = {
- name: 'Microsoft.FinOpsHubs.Core_ADF.DeleteOldResources'
- dependsOn: [
- triggerManagerRoleAssignments
- stopTriggers
- ]
- params: {
- app: app
- identityName: triggerManagerIdentity.name
- scriptContent: loadTextContent('./scripts/Remove-OldResources.ps1')
- environmentVariables: [
- {
- name: 'DataFactorySubscriptionId'
- value: subscription().id
- }
- {
- name: 'DataFactoryResourceGroup'
- value: resourceGroup().name
- }
- {
- name: 'DataFactoryName'
- value: dataFactory.name
- }
- ]
- }
-}
-
-//------------------------------------------------------------------------------
-// Stop all triggers before deploying
-//------------------------------------------------------------------------------
-
-module stopTriggers 'hub-deploymentScript.bicep' = {
- name: 'Microsoft.FinOpsHubs.Core_ADF.StopTriggers'
- dependsOn: [
- triggerManagerRoleAssignments
- ]
- params: {
- app: app
- identityName: triggerManagerIdentity.name
- scriptContent: loadTextContent('./scripts/Start-Triggers.ps1')
- arguments: '-Stop'
- environmentVariables: [
- {
- name: 'DataFactorySubscriptionId'
- value: subscription().id
- }
- {
- name: 'DataFactoryResourceGroup'
- value: resourceGroup().name
- }
- {
- name: 'DataFactoryName'
- value: dataFactory.name
- }
- {
- name: 'Triggers'
- value: join(allHubTriggers, '|')
- }
- ]
- }
-}
-
-//------------------------------------------------------------------------------
-// Linked services
-//------------------------------------------------------------------------------
-
-// cSpell:ignore linkedservices
-// TODO: Move to the hub-app module
-resource linkedService_keyVault 'Microsoft.DataFactory/factories/linkedservices@2018-06-01' = if (!empty(remoteHubStorageUri)) {
- name: keyVault.name
- parent: dataFactory
- dependsOn: enablePublicAccess ? [] : [managedIntegrationRuntime]
- properties: {
- annotations: []
- parameters: {}
- type: 'AzureKeyVault'
- typeProperties: {
- baseUrl: reference('Microsoft.KeyVault/vaults/${keyVault.name}', '2023-02-01').vaultUri
- }
- connectVia: enablePublicAccess ? null : {
- referenceName: managedIntegrationRuntime.name
- type: 'IntegrationRuntimeReference'
- }
- }
-}
-
-// TODO: Move to the hub-app module
-resource linkedService_storageAccount 'Microsoft.DataFactory/factories/linkedservices@2018-06-01' = {
- name: storageAccount.name
- parent: dataFactory
- dependsOn: enablePublicAccess ? [] : [managedIntegrationRuntime]
- properties: {
- annotations: []
- parameters: {}
- type: 'AzureBlobFS'
- typeProperties: {
- url: reference('Microsoft.Storage/storageAccounts/${storageAccount.name}', '2021-08-01').primaryEndpoints.dfs
- }
- connectVia: enablePublicAccess ? null : {
- referenceName: managedIntegrationRuntime.name
- type: 'IntegrationRuntimeReference'
- }
- }
-}
-
-resource linkedService_dataExplorer 'Microsoft.DataFactory/factories/linkedservices@2018-06-01' = if (deployDataExplorer || useFabric) {
- name: hubDataExplorerName
- parent: dataFactory
- dependsOn: enablePublicAccess ? [] : [managedIntegrationRuntime]
- properties: {
- type: 'AzureDataExplorer'
- parameters: {
- database: {
- type: 'String'
- defaultValue: dataExplorerIngestionDatabase
- }
- }
- typeProperties: {
- endpoint: dataExplorerUri
- database: '@{linkedService().database}'
- tenant: dataFactory.identity.tenantId
- servicePrincipalId: dataFactory.identity.principalId
- }
- connectVia: enablePublicAccess ? null : {
- referenceName: managedIntegrationRuntime.name
- type: 'IntegrationRuntimeReference'
- }
- }
-}
-
-resource linkedService_remoteHubStorage 'Microsoft.DataFactory/factories/linkedservices@2018-06-01' = if (!empty(remoteHubStorageUri)) {
- name: 'remoteHubStorage'
- parent: dataFactory
- dependsOn: enablePublicAccess ? [] : [managedIntegrationRuntime]
- properties: {
- annotations: []
- parameters: {}
- type: 'AzureBlobFS'
- typeProperties: {
- url: remoteHubStorageUri
- accountKey: {
- type: 'AzureKeyVaultSecret'
- store: {
- referenceName: linkedService_keyVault.name
- type: 'LinkedServiceReference'
- }
- secretName: '${toLower(hubName)}-storage-key'
- }
- }
- connectVia: enablePublicAccess ? null : {
- referenceName: managedIntegrationRuntime.name
- type: 'IntegrationRuntimeReference'
- }
- }
-}
-
-resource linkedService_ftkRepo 'Microsoft.DataFactory/factories/linkedservices@2018-06-01' = {
- name: 'ftkRepo'
- parent: dataFactory
- dependsOn: enablePublicAccess ? [] : [managedIntegrationRuntime]
- properties: {
- parameters: {
- filePath: {
- type: 'string'
- }
- }
- annotations: []
- type: 'HttpServer'
- typeProperties: {
- url: '@concat(\'https://github.com/microsoft/finops-toolkit/\', linkedService().filePath)'
- enableServerCertificateValidation: true
- authenticationType: 'Anonymous'
- }
- connectVia: enablePublicAccess ? null : {
- referenceName: managedIntegrationRuntime.name
- type: 'IntegrationRuntimeReference'
- }
- }
-}
-
-//------------------------------------------------------------------------------
-// Datasets
-//------------------------------------------------------------------------------
-
-resource dataset_config 'Microsoft.DataFactory/factories/datasets@2018-06-01' = {
- name: safeConfigContainerName
- parent: dataFactory
- properties: {
- annotations: []
- parameters: {
- fileName: {
- type: 'String'
- defaultValue: 'settings.json'
- }
- folderPath: {
- type: 'String'
- defaultValue: configContainerName
- }
- }
- type: 'Json'
- typeProperties: datasetPropsDefault
- linkedServiceName: {
- parameters: {}
- referenceName: linkedService_storageAccount.name
- type: 'LinkedServiceReference'
- }
- }
-}
-
-resource dataset_manifest 'Microsoft.DataFactory/factories/datasets@2018-06-01' = {
- name: 'manifest'
- parent: dataFactory
- properties: {
- annotations: []
- parameters: {
- fileName: {
- type: 'String'
- defaultValue: 'manifest.json'
- }
- folderPath: {
- type: 'String'
- defaultValue: exportContainerName
- }
- }
- type: 'Json'
- typeProperties: datasetPropsDefault
- linkedServiceName: {
- parameters: {}
- referenceName: linkedService_storageAccount.name
- type: 'LinkedServiceReference'
- }
- }
-}
-
-resource dataset_msexports 'Microsoft.DataFactory/factories/datasets@2018-06-01' = {
- name: safeExportContainerName
- parent: dataFactory
- properties: {
- annotations: []
- parameters: {
- blobPath: {
- type: 'String'
- }
- }
- type: 'DelimitedText'
- typeProperties: {
- location: {
- type: 'AzureBlobFSLocation'
- fileName: {
- value: '@{dataset().blobPath}'
- type: 'Expression'
- }
- fileSystem: safeExportContainerName
- }
- columnDelimiter: ','
- escapeChar: '"'
- quoteChar: '"'
- firstRowAsHeader: true
- }
- linkedServiceName: {
- parameters: {}
- referenceName: linkedService_storageAccount.name
- type: 'LinkedServiceReference'
- }
- }
-}
-
-resource dataset_msexports_gzip 'Microsoft.DataFactory/factories/datasets@2018-06-01' = {
- name: '${safeExportContainerName}_gzip'
- parent: dataFactory
- properties: {
- annotations: []
- parameters: {
- blobPath: {
- type: 'String'
- }
- }
- type: 'DelimitedText'
- typeProperties: {
- location: {
- type: 'AzureBlobFSLocation'
- fileName: {
- value: '@{dataset().blobPath}'
- type: 'Expression'
- }
- fileSystem: safeExportContainerName
- }
- columnDelimiter: ','
- escapeChar: '"'
- quoteChar: '"'
- firstRowAsHeader: true
- compressionCodec: 'Gzip'
- }
- linkedServiceName: {
- parameters: {}
- referenceName: linkedService_storageAccount.name
- type: 'LinkedServiceReference'
- }
- }
-}
-
-resource dataset_msexports_parquet 'Microsoft.DataFactory/factories/datasets@2018-06-01' = {
- name: '${safeExportContainerName}_parquet'
- parent: dataFactory
- properties: {
- annotations: []
- parameters: {
- blobPath: {
- type: 'String'
- }
- }
- type: 'Parquet'
- typeProperties: {
- location: {
- type: 'AzureBlobFSLocation'
- fileName: {
- value: '@{dataset().blobPath}'
- type: 'Expression'
- }
- fileSystem: safeExportContainerName
- }
- }
- linkedServiceName: {
- parameters: {}
- referenceName: linkedService_storageAccount.name
- type: 'LinkedServiceReference'
- }
- }
-}
-
-resource dataset_ingestion 'Microsoft.DataFactory/factories/datasets@2018-06-01' = {
- name: safeIngestionContainerName
- parent: dataFactory
- properties: {
- annotations: []
- parameters: {
- blobPath: {
- type: 'String'
- }
- }
- type: 'Parquet'
- typeProperties: {
- location: {
- type: 'AzureBlobFSLocation'
- fileName: {
- value: '@{dataset().blobPath}'
- type: 'Expression'
- }
- fileSystem: safeIngestionContainerName
- }
- }
- linkedServiceName: {
- parameters: {}
- referenceName: empty(remoteHubStorageUri) ? linkedService_storageAccount.name : linkedService_remoteHubStorage.name
- type: 'LinkedServiceReference'
- }
- }
-}
-
-resource dataset_ingestion_files 'Microsoft.DataFactory/factories/datasets@2018-06-01' = {
- name: '${safeIngestionContainerName}_files'
- parent: dataFactory
- properties: {
- annotations: []
- parameters: {
- folderPath: {
- type: 'String'
- }
- }
- type: 'Parquet'
- typeProperties: {
- location: {
- type: 'AzureBlobFSLocation'
- fileSystem: safeIngestionContainerName
- folderPath: {
- value: '@dataset().folderPath'
- type: 'Expression'
- }
- }
- }
- linkedServiceName: {
- parameters: {}
- referenceName: empty(remoteHubStorageUri) ? linkedService_storageAccount.name : linkedService_remoteHubStorage.name
- type: 'LinkedServiceReference'
- }
- }
-}
-
-resource dataset_dataExplorer 'Microsoft.DataFactory/factories/datasets@2018-06-01' = if (deployDataExplorer || useFabric) {
- name: hubDataExplorerName
- parent: dataFactory
- properties: {
- type: 'AzureDataExplorerTable'
- linkedServiceName: {
- parameters: {
- database: '@dataset().database'
- }
- referenceName: linkedService_dataExplorer.name
- type: 'LinkedServiceReference'
- }
- parameters: {
- database: {
- type: 'String'
- defaultValue: dataExplorerIngestionDatabase
- }
- table: { type: 'String' }
- }
- typeProperties: {
- table: {
- value: '@dataset().table'
- type: 'Expression'
- }
- }
- }
-}
-
-resource dataset_ftkReleaseFile 'Microsoft.DataFactory/factories/datasets@2018-06-01' = {
- name: 'ftkReleaseFile'
- parent: dataFactory
- properties: {
- linkedServiceName: {
- referenceName: linkedService_ftkRepo.name
- type: 'LinkedServiceReference'
- }
- parameters: {
- fileName: {
- type: 'string'
- }
- version: {
- type: 'string'
- defaultValue: ftkVersion
- }
- }
- annotations: []
- type: 'DelimitedText'
- typeProperties: {
- location: {
- type: 'HttpServerLocation'
- relativeUrl: {
- value: '@concat(\'releases/download/v\', dataset().version, \'/\', dataset().fileName)'
- type: 'Expression'
- }
- }
- columnDelimiter: ','
- escapeChar: '\\'
- firstRowAsHeader: true
- quoteChar: '"'
- }
- schema: []
- }
-}
-
-//------------------------------------------------------------------------------
-// Triggers
-//------------------------------------------------------------------------------
-
-// TODO: Create apps_PublishEvent pipeline { event, properties }
-
-module trigger_ExportManifestAdded 'hub-event-trigger.bicep' = {
- name: 'Microsoft.FinOpsHubs.Core_ExportManifestAddedTrigger'
- dependsOn: [
- stopTriggers
- ]
- params: {
- dataFactoryName: dataFactory.name
- triggerName: exportManifestAddedTriggerName
-
- // TODO: Replace pipeline with event: 'Microsoft.CostManagement.Exports.ManifestAdded'
- pipelineName: pipeline_ExecuteExportsETL.name
- pipelineParameters: {
- folderPath: '@triggerBody().folderPath'
- fileName: '@triggerBody().fileName'
- }
-
- storageAccountName: storageAccount.name
- storageContainer: exportContainerName
- storagePathEndsWith: 'manifest.json'
- }
-}
-
-module trigger_IngestionManifestAdded 'hub-event-trigger.bicep' = if (deployDataExplorer || useFabric) {
- name: 'Microsoft.FinOpsHubs.Core_IngestionManifestAddedTrigger'
- dependsOn: [
- stopTriggers
- ]
- params: {
- dataFactoryName: dataFactory.name
- triggerName: ingestionManifestAddedTriggerName
-
- // TODO: Replace pipeline with event: 'Microsoft.FinOpsHubs.Core.IngestionManifestAdded'
- pipelineName: pipeline_ExecuteIngestionETL.name
- pipelineParameters: {
- folderPath: '@triggerBody().folderPath'
- }
-
- storageAccountName: storageAccount.name
- storageContainer: ingestionContainerName
- storagePathEndsWith: 'manifest.json'
- }
-}
-
-module trigger_SettingsUpdated 'hub-event-trigger.bicep' = if (enableManagedExports) {
- name: 'Microsoft.FinOpsHubs.Core_SettingsUpdatedTrigger'
- dependsOn: [
- stopTriggers
- ]
- params: {
- dataFactoryName: dataFactory.name
- triggerName: updateConfigTriggerName
-
- // TODO: Replace pipeline with event: 'Microsoft.FinOpsHubs.Core.SettingsUpdated'
- pipelineName: pipeline_ConfigureExports.name
- pipelineParameters: {}
-
- storageAccountName: storageAccount.name
- storageContainer: configContainerName
- // TODO: Change this to startswith
- storagePathEndsWith: 'settings.json'
- }
-}
-
-resource trigger_DailySchedule 'Microsoft.DataFactory/factories/triggers@2018-06-01' = if (enableManagedExports) {
- name: dailyTriggerName
- parent: dataFactory
- dependsOn: [
- stopTriggers
- ]
- properties: {
- pipelines: [
- {
- pipelineReference: {
- referenceName: pipeline_StartExportProcess.name
- type: 'PipelineReference'
- }
- parameters: {
- Recurrence: 'Daily'
- }
- }
- ]
- type: 'ScheduleTrigger'
- typeProperties: {
- recurrence: {
- frequency: 'Hour'
- interval: 24
- startTime: '2023-01-01T01:01:00'
- timeZone: azuretimezones.outputs.Timezone
- }
- }
- }
-}
-
-resource trigger_MonthlySchedule 'Microsoft.DataFactory/factories/triggers@2018-06-01' = if (enableManagedExports) {
- name: monthlyTriggerName
- parent: dataFactory
- dependsOn: [
- stopTriggers
- ]
- properties: {
- pipelines: [
- {
- pipelineReference: {
- referenceName: pipeline_StartExportProcess.name
- type: 'PipelineReference'
- }
- parameters: {
- Recurrence: 'Monthly'
- }
- }
- ]
- type: 'ScheduleTrigger'
- typeProperties: {
- recurrence: {
- frequency: 'Month'
- interval: 1
- startTime: '2023-01-05T01:11:00'
- timeZone: azuretimezones.outputs.Timezone
- schedule: {
- monthDays: [
- 2
- 5
- 19
- ]
- }
- }
- }
- }
-}
-
-//------------------------------------------------------------------------------
-// Pipelines
-//------------------------------------------------------------------------------
-
-//------------------------------------------------------------------------------
-// config_InitializeHub pipeline
-//------------------------------------------------------------------------------
-@description('Initializes the hub instance based on the configuration settings.')
-resource pipeline_InitializeHub 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = if (deployDataExplorer || useFabric) {
- name: '${safeConfigContainerName}_InitializeHub'
- parent: dataFactory
- properties: {
- activities: [
- { // Get Config
- name: 'Get Config'
- type: 'Lookup'
- dependsOn: []
- policy: {
- timeout: '0.00:05:00'
- retry: 2
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- source: {
- type: 'JsonSource'
- storeSettings: {
- type: 'AzureBlobFSReadSettings'
- recursive: true
- enablePartitionDiscovery: false
- }
- formatSettings: {
- type: 'JsonReadSettings'
- }
- }
- dataset: {
- referenceName: dataset_config.name
- type: 'DatasetReference'
- }
- }
- }
- { // Set Version
- name: 'Set Version'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Get Config'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- variableName: 'version'
- value: {
- value: '@activity(\'Get Config\').output.firstRow.version'
- type: 'Expression'
- }
- }
- }
- { // Set Scopes
- name: 'Set Scopes'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Get Config'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- variableName: 'scopes'
- value: {
- value: '@string(activity(\'Get Config\').output.firstRow.scopes)'
- type: 'Expression'
- }
- }
- }
- { // Set Retention
- name: 'Set Retention'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Get Config'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- variableName: 'retention'
- value: {
- value: '@string(activity(\'Get Config\').output.firstRow.retention)'
- type: 'Expression'
- }
- }
- }
- { // Until Capacity Is Available
- name: 'Until Capacity Is Available'
- type: 'Until'
- dependsOn: [
- {
- activity: 'Set Version'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- {
- activity: 'Set Scopes'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- {
- activity: 'Set Retention'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- expression: {
- value: '@equals(variables(\'tryAgain\'), false)'
- type: 'Expression'
- }
- activities: [
- { // Confirm Ingestion Capacity
- name: 'Confirm Ingestion Capacity'
- type: 'AzureDataExplorerCommand'
- dependsOn: []
- policy: {
- timeout: '0.12:00:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- // cSpell:ignore Ingestions
- command: '.show capacity | where Resource == \'Ingestions\' | project Remaining'
- commandTimeout: '00:20:00'
- }
- linkedServiceName: {
- referenceName: linkedService_dataExplorer.name
- type: 'LinkedServiceReference'
- parameters: {
- database: dataExplorerIngestionDatabase
- }
- }
- }
- { // If Has Capacity
- name: 'If Has Capacity'
- type: 'IfCondition'
- dependsOn: [
- {
- activity: 'Confirm Ingestion Capacity'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- expression: {
- value: '@or(equals(activity(\'Confirm Ingestion Capacity\').output.count, 0), greater(activity(\'Confirm Ingestion Capacity\').output.value[0].Remaining, 0))'
- type: 'Expression'
- }
- ifFalseActivities: [
- { // Wait for Ingestion
- name: 'Wait for Ingestion'
- type: 'Wait'
- dependsOn: []
- userProperties: []
- typeProperties: {
- waitTimeInSeconds: 15
- }
- }
- { // Try Again
- name: 'Try Again'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Wait for Ingestion'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'tryAgain'
- value: true
- }
- }
- ]
- ifTrueActivities: [
- { // Save ingestion policy in ADX
- name: 'Set ingestion policy in ADX'
- type: 'AzureDataExplorerCommand'
- dependsOn: []
- policy: {
- timeout: '0.12:00:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- command: {
- // Do not attempt to set the ingestion policy if using Fabric; use a simple query as a placeholder
- value: useFabric
- ? '.show database ${dataExplorerIngestionDatabase} policy managed_identity'
- : '.alter-merge database ${dataExplorerIngestionDatabase} policy managed_identity "[ { \'ObjectId\' : \'${dataExplorerPrincipalId}\', \'AllowedUsages\' : \'NativeIngestion\' }]"'
- type: 'Expression'
- }
- commandTimeout: '00:20:00'
- }
- linkedServiceName: {
- referenceName: linkedService_dataExplorer.name
- type: 'LinkedServiceReference'
- parameters: {
- database: dataExplorerIngestionDatabase
- }
- }
- }
- { // Save Hub Settings in ADX
- name: 'Save Hub Settings in ADX'
- type: 'AzureDataExplorerCommand'
- dependsOn: [
- {
- activity: 'Set ingestion policy in ADX'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- timeout: '0.12:00:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- command: {
- // cSpell:ignore isnull, isnotempty
- value: '@concat(\'.append HubSettingsLog <| print version="\', variables(\'version\'), \'",scopes=dynamic(\', variables(\'scopes\'), \'),retention=dynamic(\', variables(\'retention\'), \') | extend scopes = iff(isnull(scopes[0]), pack_array(scopes), scopes) | mv-apply scopeObj = scopes on (where isnotempty(scopeObj.scope) | summarize scopes = make_set(scopeObj.scope))\')'
- type: 'Expression'
- }
- commandTimeout: '00:20:00'
- }
- linkedServiceName: {
- referenceName: linkedService_dataExplorer.name
- type: 'LinkedServiceReference'
- parameters: {
- database: dataExplorerIngestionDatabase
- }
- }
- }
- { // Update PricingUnits in ADX
- name: 'Update PricingUnits in ADX'
- type: 'AzureDataExplorerCommand'
- dependsOn: [
- {
- activity: 'Save Hub Settings in ADX'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- timeout: '0.12:00:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- // cSpell:ignore externaldata
- command: '.set-or-replace PricingUnits <| externaldata(x_PricingUnitDescription: string, AccountTypes: string, x_PricingBlockSize: decimal, PricingUnit: string)[@"${ftkReleaseUri}/PricingUnits.csv"] with (format="csv", ignoreFirstRecord=true) | project-away AccountTypes'
- commandTimeout: '00:20:00'
- }
- linkedServiceName: {
- referenceName: linkedService_dataExplorer.name
- type: 'LinkedServiceReference'
- parameters: {
- database: dataExplorerIngestionDatabase
- }
- }
- }
- { // Update Regions in ADX
- name: 'Update Regions in ADX'
- type: 'AzureDataExplorerCommand'
- dependsOn: [
- {
- activity: 'Update PricingUnits in ADX'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- timeout: '0.12:00:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- command: '.set-or-replace Regions <| externaldata(ResourceLocation: string, RegionId: string, RegionName: string)[@"${ftkReleaseUri}/Regions.csv"] with (format="csv", ignoreFirstRecord=true)'
- commandTimeout: '00:20:00'
- }
- linkedServiceName: {
- referenceName: linkedService_dataExplorer.name
- type: 'LinkedServiceReference'
- parameters: {
- database: dataExplorerIngestionDatabase
- }
- }
- }
- { // Update ResourceTypes in ADX
- name: 'Update ResourceTypes in ADX'
- type: 'AzureDataExplorerCommand'
- dependsOn: [
- {
- activity: 'Update Regions in ADX'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- timeout: '0.12:00:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- command: '.set-or-replace ResourceTypes <| externaldata(x_ResourceType: string, SingularDisplayName: string, PluralDisplayName: string, LowerSingularDisplayName: string, LowerPluralDisplayName: string, IsPreview: bool, Description: string, IconUri: string, Links: string)[@"${ftkReleaseUri}/ResourceTypes.csv"] with (format="csv", ignoreFirstRecord=true) | project-away Links'
- commandTimeout: '00:20:00'
- }
- linkedServiceName: {
- referenceName: linkedService_dataExplorer.name
- type: 'LinkedServiceReference'
- parameters: {
- database: dataExplorerIngestionDatabase
- }
- }
- }
- { // Update Services in ADX
- name: 'Update Services in ADX'
- type: 'AzureDataExplorerCommand'
- dependsOn: [
- {
- activity: 'Update ResourceTypes in ADX'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- timeout: '0.12:00:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- command: '.set-or-replace Services <| externaldata(x_ConsumedService: string, x_ResourceType: string, ServiceName: string, ServiceCategory: string, ServiceSubcategory: string, PublisherName: string, x_PublisherCategory: string, x_Environment: string, x_ServiceModel: string)[@"${ftkReleaseUri}/Services.csv"] with (format="csv", ignoreFirstRecord=true)'
- commandTimeout: '00:20:00'
- }
- linkedServiceName: {
- referenceName: linkedService_dataExplorer.name
- type: 'LinkedServiceReference'
- parameters: {
- database: dataExplorerIngestionDatabase
- }
- }
- }
- { // Ingestion Complete
- name: 'Ingestion Complete'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Update Services in ADX'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'tryAgain'
- value: false
- }
- }
- ]
- }
- }
- { // Abort On Error
- name: 'Abort On Error'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'If Has Capacity'
- dependencyConditions: [
- 'Failed'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'tryAgain'
- value: false
- }
- }
- ]
- timeout: '0.02:00:00'
- }
- }
- { // Timeout Error
- name: 'Timeout Error'
- type: 'Fail'
- dependsOn: [
- {
- activity: 'Until Capacity Is Available'
- dependencyConditions: [
- 'Failed'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- message: 'Data Explorer ingestion timed out after 2 hours while waiting for available capacity. Please re-run this pipeline to re-attempt ingestion. If you continue to see this error, please report an issue at https://aka.ms/ftk/ideas.'
- errorCode: 'DataExplorerIngestionTimeout'
- }
- }
- ]
- concurrency: 1
- variables: {
- version: {
- type: 'String'
- }
- scopes: {
- type: 'String'
- }
- retention: {
- type: 'String'
- }
- tryAgain: {
- type: 'Boolean'
- defaultValue: true
- }
- }
- }
-}
-
-//------------------------------------------------------------------------------
-// config_StartBackfillProcess pipeline
-//------------------------------------------------------------------------------
-@description('Runs the backfill job for each month based on retention settings.')
-resource pipeline_StartBackfillProcess 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = if (enableManagedExports) {
- name: '${safeConfigContainerName}_StartBackfillProcess'
- parent: dataFactory
- properties: {
- activities: [
- { // Get Config
- name: 'Get Config'
- type: 'Lookup'
- dependsOn: []
- policy: {
- timeout: '0.00:05:00'
- retry: 2
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- source: {
- type: 'JsonSource'
- storeSettings: {
- type: 'AzureBlobFSReadSettings'
- recursive: true
- enablePartitionDiscovery: false
- }
- formatSettings: {
- type: 'JsonReadSettings'
- }
- }
- dataset: {
- referenceName: dataset_config.name
- type: 'DatasetReference'
- parameters: {
- fileName: {
- value: '@variables(\'fileName\')'
- type: 'Expression'
- }
- folderPath: {
- value: '@variables(\'folderPath\')'
- type: 'Expression'
- }
- }
- }
- }
- }
- { // Set backfill end date
- name: 'Set backfill end date'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Get Config'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- variableName: 'endDate'
- value: {
- value: '@addDays(startOfMonth(utcNow()), -1)'
- type: 'Expression'
- }
- }
- }
- { // Set backfill start date
- name: 'Set backfill start date'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Get Config'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- variableName: 'startDate'
- value: {
- value: '@subtractFromTime(startOfMonth(utcNow()), activity(\'Get Config\').output.firstRow.retention.ingestion.months, \'Month\')'
- type: 'Expression'
- }
- }
- }
- { // Set export start date
- name: 'Set export start date'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Set backfill start date'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- variableName: 'thisMonth'
- value: {
- value: '@startOfMonth(variables(\'endDate\'))'
- type: 'Expression'
- }
- }
- }
- { // Set export end date
- name: 'Set export end date'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Set export start date'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- variableName: 'nextMonth'
- value: {
- value: '@startOfMonth(subtractFromTime(variables(\'thisMonth\'), 1, \'Month\'))'
- type: 'Expression'
- }
- }
- }
- { // Every Month
- name: 'Every Month'
- type: 'Until'
- dependsOn: [
- {
- activity: 'Set export end date'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- {
- activity: 'Set backfill end date'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- expression: {
- value: '@less(variables(\'thisMonth\'), variables(\'startDate\'))'
- type: 'Expression'
- }
- activities: [
- {
- name: 'Update export start date'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Backfill data'
- dependencyConditions: [
- 'Completed'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- variableName: 'thisMonth'
- value: {
- value: '@variables(\'nextMonth\')'
- type: 'Expression'
- }
- }
- }
- {
- name: 'Update export end date'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Update export start date'
- dependencyConditions: [
- 'Completed'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- variableName: 'nextMonth'
- value: {
- value: '@subtractFromTime(variables(\'thisMonth\'), 1, \'Month\')'
- type: 'Expression'
- }
- }
- }
- {
- name: 'Backfill data'
- type: 'ExecutePipeline'
- dependsOn: []
- userProperties: []
- typeProperties: {
- pipeline: {
- referenceName: pipeline_RunBackfillJob.name
- type: 'PipelineReference'
- }
- waitOnCompletion: true
- parameters: {
- StartDate: {
- value: '@variables(\'thisMonth\')'
- type: 'Expression'
- }
- EndDate: {
- value: '@addDays(addToTime(variables(\'thisMonth\'), 1, \'Month\'), -1)'
- type: 'Expression'
- }
- }
- }
- }
- ]
- timeout: '0.02:00:00'
- }
- }
- ]
- concurrency: 1
- variables: {
- exportName: {
- type: 'String'
- }
- storageAccountId: {
- type: 'String'
- defaultValue: storageAccount.id
- }
- finOpsHub: {
- type: 'String'
- defaultValue: hubName
- }
- resourceManagementUri: {
- type: 'String'
- defaultValue: environment().resourceManager
- }
- fileName: {
- type: 'String'
- defaultValue: 'settings.json'
- }
- folderPath: {
- type: 'String'
- defaultValue: configContainerName
- }
- endDate: {
- type: 'String'
- }
- startDate: {
- type: 'String'
- }
- thisMonth: {
- type: 'String'
- }
- nextMonth: {
- type: 'String'
- }
- }
- }
-}
-
-//------------------------------------------------------------------------------
-// config_RunBackfillJob pipeline
-// Triggered by config_StartBackfillProcess pipeline
-//------------------------------------------------------------------------------
-@description('Creates and triggers exports for all defined scopes for the specified date range.')
-resource pipeline_RunBackfillJob 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = if (enableManagedExports) {
- name: '${safeConfigContainerName}_RunBackfillJob'
- parent: dataFactory
- properties: {
- activities: [
- { // Get Config
- name: 'Get Config'
- type: 'Lookup'
- dependsOn: []
- policy: {
- timeout: '0.00:05:00'
- retry: 2
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- source: {
- type: 'JsonSource'
- storeSettings: {
- type: 'AzureBlobFSReadSettings'
- recursive: true
- enablePartitionDiscovery: false
- }
- formatSettings: {
- type: 'JsonReadSettings'
- }
- }
- dataset: {
- referenceName: dataset_config.name
- type: 'DatasetReference'
- parameters: {
- fileName: {
- value: '@variables(\'fileName\')'
- type: 'Expression'
- }
- folderPath: {
- value: '@variables(\'folderPath\')'
- type: 'Expression'
- }
- }
- }
- }
- }
- { // Set Scopes
- name: 'Set Scopes'
- description: 'Save scopes to test if it is an array'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Get Config'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'scopesArray'
- value: {
- value: '@activity(\'Get Config\').output.firstRow.scopes'
- type: 'Expression'
- }
- }
- }
- { // Set Scopes as Array
- name: 'Set Scopes as Array'
- description: 'Wraps a single scope object into an array to work around the PowerShell bug where single-item arrays are sometimes written as a single object instead of an array.'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Set Scopes'
- dependencyConditions: [
- 'Failed'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'scopesArray'
- value: {
- value: '@createArray(activity(\'Get Config\').output.firstRow.scopes)'
- type: 'Expression'
- }
- }
- }
- { // Filter Invalid Scopes
- name: 'Filter Invalid Scopes'
- description: 'Remove any invalid scopes to avoid errors.'
- type: 'Filter'
- dependsOn: [
- {
- activity: 'Set Scopes'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- {
- activity: 'Set Scopes as Array'
- dependencyConditions: [
- 'Skipped'
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- items: {
- value: '@variables(\'scopesArray\')'
- type: 'Expression'
- }
- condition: {
- value: '@and(not(empty(item().scope)), not(equals(item().scope, \'/\')))'
- type: 'Expression'
- }
- }
- }
- { // ForEach Export Scope
- name: 'ForEach Export Scope'
- type: 'ForEach'
- dependsOn: [
- {
- activity: 'Filter Invalid Scopes'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- items: {
- value: '@activity(\'Filter Invalid Scopes\').output.Value'
- type: 'Expression'
- }
- isSequential: true
- activities: [
- {
- name: 'Set backfill export name'
- type: 'SetVariable'
- dependsOn: []
- userProperties: []
- typeProperties: {
- variableName: 'exportName'
- value: {
- // cSpell:ignore costdetails
- value: '@toLower(concat(variables(\'finOpsHub\'), \'-monthly-costdetails\'))'
- type: 'Expression'
- }
- }
- }
- {
- name: 'Trigger backfill export'
- type: 'WebActivity'
- dependsOn: [
- {
- activity: 'Set backfill export name'
- dependencyConditions: [
- 'Completed'
- ]
- }
- ]
- policy: {
- timeout: '0.00:05:00'
- retry: 1
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- url: {
- value: '@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{variables(\'exportName\')}/run?api-version=${exportApiVersion}'
- type: 'Expression'
- }
- method: 'POST'
- headers: {
- 'x-ms-command-name': 'FinOpsToolkit.Hubs.config_RunBackfill@${ftkVersion}'
- 'Content-Type': 'application/json'
- ClientType: 'FinOpsToolkit.Hubs@${ftkVersion}'
- }
- body: '{"timePeriod" : { "from" : "@{pipeline().parameters.StartDate}", "to" : "@{pipeline().parameters.EndDate}" }}'
- authentication: {
- type: 'MSI'
- resource: {
- value: '@variables(\'resourceManagementUri\')'
- type: 'Expression'
- }
- }
- }
- }
- ]
- }
- }
- ]
- concurrency: 1
- parameters: {
- StartDate: {
- type: 'string'
- }
- EndDate: {
- type: 'string'
- }
- }
- variables: {
- exportName: {
- type: 'String'
- }
- storageAccountId: {
- type: 'String'
- defaultValue: storageAccount.id
- }
- finOpsHub: {
- type: 'String'
- defaultValue: hubName
- }
- resourceManagementUri: {
- type: 'String'
- defaultValue: environment().resourceManager
- }
- fileName: {
- type: 'String'
- defaultValue: 'settings.json'
- }
- folderPath: {
- type: 'String'
- defaultValue: configContainerName
- }
- scopesArray: {
- type: 'Array'
- }
- }
- }
-}
-
-//------------------------------------------------------------------------------
-// config_StartExportProcess pipeline
-// Triggered by config_DailySchedule/MonthlySchedule triggers
-//------------------------------------------------------------------------------
-@description('Gets a list of all Cost Management exports configured for this hub based on the scopes defined in settings.json, then runs each export using the config_RunExportJobs pipeline.')
-resource pipeline_StartExportProcess 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = if (enableManagedExports) {
- name: '${safeConfigContainerName}_StartExportProcess'
- parent: dataFactory
- properties: {
- activities: [
- { // Get Config
- name: 'Get Config'
- type: 'Lookup'
- dependsOn: []
- policy: {
- timeout: '0.00:05:00'
- retry: 2
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- source: {
- type: 'JsonSource'
- storeSettings: {
- type: 'AzureBlobFSReadSettings'
- recursive: true
- enablePartitionDiscovery: false
- }
- formatSettings: {
- type: 'JsonReadSettings'
- }
- }
- dataset: {
- referenceName: dataset_config.name
- type: 'DatasetReference'
- parameters: {
- fileName: {
- value: '@variables(\'fileName\')'
- type: 'Expression'
- }
- folderPath: {
- value: '@variables(\'folderPath\')'
- type: 'Expression'
- }
- }
- }
- }
- }
- { // Set Scopes
- name: 'Set Scopes'
- description: 'Save scopes to test if it is an array'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Get Config'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'scopesArray'
- value: {
- value: '@activity(\'Get Config\').output.firstRow.scopes'
- type: 'Expression'
- }
- }
- }
- { // Set Scopes as Array
- name: 'Set Scopes as Array'
- description: 'Wraps a single scope object into an array to work around the PowerShell bug where single-item arrays are sometimes written as a single object instead of an array.'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Set Scopes'
- dependencyConditions: [
- 'Failed'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'scopesArray'
- value: {
- value: '@createArray(activity(\'Get Config\').output.firstRow.scopes)'
- type: 'Expression'
- }
- }
- }
- { // Filter Invalid Scopes
- name: 'Filter Invalid Scopes'
- description: 'Remove any invalid scopes to avoid errors.'
- type: 'Filter'
- dependsOn: [
- {
- activity: 'Set Scopes'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- {
- activity: 'Set Scopes as Array'
- dependencyConditions: [
- 'Succeeded'
- 'Skipped'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- items: {
- value: '@variables(\'scopesArray\')'
- type: 'Expression'
- }
- condition: {
- value: '@and(not(empty(item().scope)), not(equals(item().scope, \'/\')))'
- type: 'Expression'
- }
- }
- }
- { // ForEach Export Scope
- name: 'ForEach Export Scope'
- type: 'ForEach'
- dependsOn: [
- {
- activity: 'Filter Invalid Scopes'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- items: {
- value: '@activity(\'Filter Invalid Scopes\').output.Value'
- type: 'Expression'
- }
- isSequential: true
- activities: [
- {
- name: 'Get exports for scope'
- type: 'WebActivity'
- dependsOn: []
- policy: {
- timeout: '0.00:05:00'
- retry: 2
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- url: {
- value: '@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports?api-version=${exportApiVersion}'
- type: 'Expression'
- }
- method: 'GET'
- authentication: {
- type: 'MSI'
- resource: {
- value: '@variables(\'resourceManagementUri\')'
- type: 'Expression'
- }
- }
- }
- }
- {
- name: 'Run exports for scope'
- type: 'ExecutePipeline'
- dependsOn: [
- {
- activity: 'Get exports for scope'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- pipeline: {
- referenceName: pipeline_RunExportJobs.name
- type: 'PipelineReference'
- }
- waitOnCompletion: true
- parameters: {
- ExportScopes: {
- value: '@activity(\'Get exports for scope\').output.value'
- type: 'Expression'
- }
- Recurrence: {
- value: '@pipeline().parameters.Recurrence'
- type: 'Expression'
- }
- }
- }
- }
- ]
- }
- }
- ]
- concurrency: 1
- parameters: {
- Recurrence: {
- type: 'string'
- defaultValue: 'Daily'
- }
- }
- variables: {
- fileName: {
- type: 'String'
- defaultValue: 'settings.json'
- }
- folderPath: {
- type: 'String'
- defaultValue: configContainerName
- }
- finOpsHub: {
- type: 'String'
- defaultValue: hubName
- }
- resourceManagementUri: {
- type: 'String'
- defaultValue: environment().resourceManager
- }
- scopesArray: {
- type: 'Array'
- }
- }
- }
-}
-
-//------------------------------------------------------------------------------
-// config_RunExportJobs pipeline
-// Triggered by pipeline_StartExportProcess pipeline
-//------------------------------------------------------------------------------
-@description('Runs the specified Cost Management exports.')
-resource pipeline_RunExportJobs 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = if (enableManagedExports) {
- name: '${safeConfigContainerName}_RunExportJobs'
- parent: dataFactory
- dependsOn: [
- dataset_config
- ]
- properties: {
- activities: [
- {
- name: 'ForEach export scope'
- type: 'ForEach'
- dependsOn: []
- userProperties: []
- typeProperties: {
- items: {
- value: '@pipeline().parameters.exportScopes'
- type: 'Expression'
- }
- isSequential: true
- activities: [
- {
- name: 'If scheduled'
- type: 'IfCondition'
- dependsOn: []
- userProperties: []
- typeProperties: {
- expression: {
- value: '@and( startswith(toLower(item().name), toLower(variables(\'hubName\'))), and(contains(string(item().properties.schedule), \'recurrence\'), equals(toLower(item().properties.schedule.recurrence), toLower(pipeline().parameters.Recurrence))))'
- type: 'Expression'
- }
- ifTrueActivities: [
- {
- name: 'Trigger export'
- type: 'WebActivity'
- dependsOn: []
- policy: {
- timeout: '0.00:05:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- method: 'POST'
- url: {
- value: '@{replace(toLower(concat(variables(\'resourceManagementUri\'),item().id)), \'com//\', \'com/\')}/run?api-version=${exportApiVersion}'
- type: 'Expression'
- }
- headers: {
- 'x-ms-command-name': 'FinOpsToolkit.Hubs.config_RunExportJobs@${ftkVersion}'
- ClientType: 'FinOpsToolkit.Hubs@${ftkVersion}'
- }
- body: ' '
- authentication: {
- type: 'MSI'
- resource: {
- value: '@variables(\'resourceManagementUri\')'
- type: 'Expression'
- }
- }
- }
- }
- ]
- }
- }
- ]
- }
- }
- ]
- concurrency: 1
- parameters: {
- ExportScopes: {
- type: 'array'
- }
- Recurrence: {
- type: 'string'
- defaultValue: 'Daily'
- }
- }
- variables: {
- resourceManagementUri: {
- type: 'String'
- defaultValue: environment().resourceManager
- }
- hubName: {
- type: 'String'
- defaultValue: hubName
- }
- }
- }
-}
-
-//------------------------------------------------------------------------------
-// config_ConfigureExports pipeline
-// Triggered by config_SettingsUpdated trigger
-//------------------------------------------------------------------------------
-@description('Creates Cost Management exports for supported scopes.')
-resource pipeline_ConfigureExports 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = if (enableManagedExports) {
- name: '${safeConfigContainerName}_ConfigureExports'
- parent: dataFactory
- properties: {
- activities: [
- { // Get Config
- name: 'Get Config'
- type: 'Lookup'
- dependsOn: []
- policy: {
- timeout: '0.00:05:00'
- retry: 2
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- source: {
- type: 'JsonSource'
- storeSettings: {
- type: 'AzureBlobFSReadSettings'
- recursive: true
- enablePartitionDiscovery: false
- }
- formatSettings: {
- type: 'JsonReadSettings'
- }
- }
- dataset: {
- referenceName: dataset_config.name
- type: 'DatasetReference'
- parameters: {
- fileName: {
- value: '@variables(\'fileName\')'
- type: 'Expression'
- }
- folderPath: {
- value: '@variables(\'folderPath\')'
- type: 'Expression'
- }
- }
- }
- }
- }
- { // Save Scopes
- name: 'Save Scopes'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Get Config'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'scopesArray'
- value: {
- value: '@activity(\'Get Config\').output.firstRow.scopes'
- type: 'Expression'
- }
- }
- }
- { // Save Scopes as Array
- name: 'Save Scopes as Array'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Save Scopes'
- dependencyConditions: [
- 'Failed'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'scopesArray'
- value: {
- value: '@array(activity(\'Get Config\').output.firstRow.scopes)'
- type: 'Expression'
- }
- }
- }
- { // Filter Invalid Scopes
- name: 'Filter Invalid Scopes'
- type: 'Filter'
- dependsOn: [
- {
- activity: 'Save Scopes'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- {
- activity: 'Save Scopes as Array'
- dependencyConditions: [
- 'Skipped'
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- items: {
- value: '@variables(\'scopesArray\')'
- type: 'Expression'
- }
- condition: {
- value: '@and(not(empty(item().scope)), not(equals(item().scope, \'/\')))'
- type: 'Expression'
- }
- }
- }
- { // ForEach Export Scope
- name: 'ForEach Export Scope'
- type: 'ForEach'
- dependsOn: [
- {
- activity: 'Filter Invalid Scopes'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- items: {
- value: '@activity(\'Filter Invalid Scopes\').output.value'
- type: 'Expression'
- }
- isSequential: true
- activities: [
- {
- name: 'Set Export Type'
- type: 'SetVariable'
- dependsOn: []
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'exportScopeType'
- value: {
- value: '@if(contains(toLower(item().scope), \'providers/microsoft.billing/billingaccounts\'), if(contains(toLower(item().scope), \':\'), \'mca\', \'ea\'), if(contains(toLower(item().scope), \'subscriptions/\'), \'subscription\', \'undefined\'))'
- type: 'Expression'
- }
- }
- }
- {
- name: 'Switch Export Type'
- type: 'Switch'
- dependsOn: [
- {
- activity: 'Set Export Type'
- dependencyConditions: [ 'Succeeded' ]
- }
- ]
- userProperties: []
- typeProperties: {
- on: {
- value: '@toLower(variables(\'exportScopeType\'))'
- type: 'Expression'
- }
- cases: [
- { // EA
- value: 'ea'
- activities: [
- { // 'EA open month focus export'
- name: 'EA open month focus export'
- type: 'WebActivity'
- dependsOn: [
- ]
- policy: {
- timeout: '0.00:05:00'
- retry: 2
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- url: {
- value: '@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{toLower(concat(variables(\'finOpsHub\'), \'-daily-costdetails\'))}?api-version=${exportApiVersion}'
- type: 'Expression'
- }
- method: 'PUT'
- body: {
- value: getExportBodyV2(exportContainerName, 'FocusCost', focusSchemaVersion, false, 'Parquet', 'Snappy', 'true', 'CreateNewReport', '', '', '')
- type: 'Expression'
- }
- headers: {
- 'x-ms-command-name': 'FinOpsToolkit.Hubs.config_RunExportJobs.CostsDaily@${ftkVersion}'
- ClientType: 'FinOpsToolkit.Hubs@${ftkVersion}'
- }
- authentication: {
- type: 'MSI'
- resource: {
- value: '@variables(\'resourceManagementUri\')'
- type: 'Expression'
- }
- }
- }
- }
- { // 'EA closed month focus export'
- name: 'EA closed month focus export'
- type: 'WebActivity'
- dependsOn: [
- {
- activity: 'EA open month focus export'
- dependencyConditions: [ 'Succeeded' ]
- }
- ]
- policy: {
- timeout: '0.00:05:00'
- retry: 2
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- url: {
- value: '@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{toLower(concat(variables(\'finOpsHub\'), \'-monthly-costdetails\'))}?api-version=${exportApiVersion}'
- type: 'Expression'
- }
- method: 'PUT'
- body: {
- value: getExportBodyV2(exportContainerName, 'FocusCost', focusSchemaVersion, true, 'Parquet', 'Snappy', 'true', 'CreateNewReport', '', '', '')
- type: 'Expression'
- }
- headers: {
- 'x-ms-command-name': 'FinOpsToolkit.Hubs.config_RunExportJobs.CostsMonthly@${ftkVersion}'
- ClientType: 'FinOpsToolkit.Hubs@${ftkVersion}'
- }
- authentication: {
- type: 'MSI'
- resource: {
- value: '@variables(\'resourceManagementUri\')'
- type: 'Expression'
- }
- }
- }
- }
- { // 'EA monthly pricesheet export'
- name: 'EA monthly pricesheet export'
- type: 'WebActivity'
- dependsOn: [
- {
- activity: 'EA closed month focus export'
- dependencyConditions: [ 'Succeeded' ]
- }
- ]
- policy: {
- timeout: '0.00:05:00'
- retry: 2
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- url: {
- value: '@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{toLower(concat(variables(\'finOpsHub\'), \'-monthly-pricesheet\'))}?api-version=${exportApiVersion}'
- type: 'Expression'
- }
- method: 'PUT'
- body: {
- value: getExportBodyV2(exportContainerName, 'Pricesheet', exportSchemaVersion, true, 'Parquet', 'Snappy', 'true', 'CreateNewReport', '', '', '')
- type: 'Expression'
- }
- headers: {
- 'x-ms-command-name': 'FinOpsToolkit.Hubs.config_RunExportJobs.Prices@${ftkVersion}'
- ClientType: 'FinOpsToolkit.Hubs@${ftkVersion}'
- }
- authentication: {
- type: 'MSI'
- resource: {
- value: '@variables(\'resourceManagementUri\')'
- type: 'Expression'
- }
- }
- }
- }
- {
- name: 'Trigger EA monthly pricesheet export'
- type: 'WebActivity'
- dependsOn: [
- {
- activity: 'EA monthly pricesheet export'
- dependencyConditions: [ 'Succeeded' ]
- }
- ]
- policy: {
- timeout: '0.00:05:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- method: 'POST'
- url: {
- value: '@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{toLower(concat(variables(\'finOpsHub\'), \'-monthly-pricesheet\'))}/run?api-version=${exportApiVersion}'
- type: 'Expression'
- }
- headers: {
- 'x-ms-command-name': 'FinOpsToolkit.Hubs.config_RunExportJobs.Prices@${ftkVersion}'
- ClientType: 'FinOpsToolkit.Hubs@${ftkVersion}'
- }
- body: ' '
- authentication: {
- type: 'MSI'
- resource: {
- value: '@variables(\'resourceManagementUri\')'
- type: 'Expression'
- }
- }
- }
- }
- { // 'EA daily reservation details export'
- name: 'EA daily reservation details export'
- type: 'WebActivity'
- dependsOn: [
- {
- activity: 'EA monthly pricesheet export'
- dependencyConditions: [ 'Succeeded' ]
- }
- ]
- policy: {
- timeout: '0.00:05:00'
- retry: 2
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- url: {
- value: '@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{toLower(concat(variables(\'finOpsHub\'), \'-daily-reservationdetails\'))}?api-version=${exportApiVersion}'
- type: 'Expression'
- }
- method: 'PUT'
- body: {
- value: getExportBodyV2(exportContainerName, 'ReservationDetails', reservationDetailsSchemaVersion, false, 'CSV', 'None', 'true', 'CreateNewReport', '', '', '')
- type: 'Expression'
- }
- headers: {
- 'x-ms-command-name': 'FinOpsToolkit.Hubs.config_RunExportJobs.ReservationDetails@${ftkVersion}'
- ClientType: 'FinOpsToolkit.Hubs@${ftkVersion}'
- }
- authentication: {
- type: 'MSI'
- resource: {
- value: '@variables(\'resourceManagementUri\')'
- type: 'Expression'
- }
- }
- }
- }
- { // 'EA daily reservation transactions export'
- name: 'EA daily reservation transactions export'
- type: 'WebActivity'
- dependsOn: [
- {
- activity: 'EA daily reservation details export'
- dependencyConditions: [ 'Succeeded' ]
- }
- ]
- policy: {
- timeout: '0.00:05:00'
- retry: 2
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- url: {
- value: '@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{toLower(concat(variables(\'finOpsHub\'), \'-daily-reservationtransactions\'))}?api-version=${exportApiVersion}'
- type: 'Expression'
- }
- method: 'PUT'
- body: {
- value: getExportBodyV2(exportContainerName, 'ReservationTransactions', exportSchemaVersion, false, 'CSV', 'None', 'true', 'CreateNewReport', '', '', '')
- type: 'Expression'
- }
- headers: {
- 'x-ms-command-name': 'FinOpsToolkit.Hubs.config_RunExportJobs.ReservationTransactions@${ftkVersion}'
- ClientType: 'FinOpsToolkit.Hubs@${ftkVersion}'
- }
- authentication: {
- type: 'MSI'
- resource: {
- value: '@variables(\'resourceManagementUri\')'
- type: 'Expression'
- }
- }
- }
- }
- { // 'EA daily recommendations shared last30day virtualmachines export'
- name: 'EA daily shared 30day virtualmachines'
- type: 'WebActivity'
- dependsOn: [
- {
- activity: 'EA daily reservation transactions export'
- dependencyConditions: [ 'Succeeded' ]
- }
- ]
- policy: {
- timeout: '0.00:05:00'
- retry: 2
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- url: {
- value: '@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{toLower(concat(variables(\'finOpsHub\'), \'-daily-recommendations-shared-last30days-virtualmachines\'))}?api-version=${exportApiVersion}'
- type: 'Expression'
- }
- method: 'PUT'
- body: {
- value: getExportBodyV2(exportContainerName, 'ReservationRecommendations', exportSchemaVersion, false, 'CSV', 'None', 'true', 'CreateNewReport', 'Shared', 'Last30Days', 'VirtualMachines')
- type: 'Expression'
- }
- headers: {
- 'x-ms-command-name': 'FinOpsToolkit.Hubs.config_RunExportJobs.ReservationRecommendations.VM.Shared.30d@${ftkVersion}'
- ClientType: 'FinOpsToolkit.Hubs@${ftkVersion}'
- }
- authentication: {
- type: 'MSI'
- resource: {
- value: '@variables(\'resourceManagementUri\')'
- type: 'Expression'
- }
- }
- }
- }
- ]
- }
- { // subscription
- value: 'subscription'
- activities: [
- { // 'Subscription open month focus export'
- name: 'Subscription open month focus export'
- type: 'WebActivity'
- dependsOn: [
- ]
- policy: {
- timeout: '0.00:05:00'
- retry: 2
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- url: {
- value: '@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{toLower(concat(variables(\'finOpsHub\'), \'-daily-costdetails\'))}?api-version=${exportApiVersion}'
- type: 'Expression'
- }
- method: 'PUT'
- body: {
- value: getExportBodyV2(exportContainerName, 'FocusCost', focusSchemaVersion, false, 'Parquet', 'Snappy', 'true', 'CreateNewReport', '', '', '')
- type: 'Expression'
- }
- headers: {
- 'x-ms-command-name': 'FinOpsToolkit.Hubs.config_RunExportJobs.CostsDaily@${ftkVersion}'
- ClientType: 'FinOpsToolkit.Hubs@${ftkVersion}'
- }
- authentication: {
- type: 'MSI'
- resource: {
- value: '@variables(\'resourceManagementUri\')'
- type: 'Expression'
- }
- }
- }
- }
- { // 'Subscription closed month focus export'
- name: 'Subscription closed month focus export'
- type: 'WebActivity'
- dependsOn: [
- {
- activity: 'Subscription open month focus export'
- dependencyConditions: [ 'Succeeded' ]
- }
- ]
- policy: {
- timeout: '0.00:05:00'
- retry: 2
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- url: {
- value: '@{variables(\'resourceManagementUri\')}@{item().scope}/providers/Microsoft.CostManagement/exports/@{toLower(concat(variables(\'finOpsHub\'), \'-monthly-costdetails\'))}?api-version=${exportApiVersion}'
- type: 'Expression'
- }
- method: 'PUT'
- body: {
- value: getExportBodyV2(exportContainerName, 'FocusCost', focusSchemaVersion, true, 'Parquet', 'Snappy', 'true', 'CreateNewReport', '', '', '')
- type: 'Expression'
- }
- headers: {
- 'x-ms-command-name': 'FinOpsToolkit.Hubs.config_RunExportJobs.CostsMonthly@${ftkVersion}'
- ClientType: 'FinOpsToolkit.Hubs@${ftkVersion}'
- }
- authentication: {
- type: 'MSI'
- resource: {
- value: '@variables(\'resourceManagementUri\')'
- type: 'Expression'
- }
- }
- }
- }
- ]
- }
- { // MCA
- value: 'mca'
- activities: [
- {
- name: 'Export Type Unsupported Error'
- type: 'Fail'
- dependsOn: []
- userProperties: []
- typeProperties: {
- message: {
- value: '@concat(\'MCA agreements are not supported for managed exports :\',variables(\'exportScope\'))'
- type: 'Expression'
- }
- errorCode: 'ExportTypeUnsupported'
- }
- }
- ]
- }
- ]
- defaultActivities: [
- {
- name: 'Export Type Not Defined Error'
- type: 'Fail'
- dependsOn: []
- userProperties: []
- typeProperties: {
- message: {
- value: '@concat(\'Unable to determine the export scope type for :\',variables(\'exportScope\'))'
- type: 'Expression'
- }
- errorCode: 'ExportTypeNotDefined'
- }
- }
- ]
- }
- }
- ]
- }
- }
- ]
- concurrency: 1
- variables: {
- scopesArray: {
- type: 'Array'
- }
- exportName: {
- type: 'String'
- }
- exportScope: {
- type: 'String'
- }
- exportScopeType: {
- type: 'String'
- }
- storageAccountId: {
- type: 'String'
- defaultValue: storageAccount.id
- }
- finOpsHub: {
- type: 'String'
- defaultValue: hubName
- }
- resourceManagementUri: {
- type: 'String'
- defaultValue: environment().resourceManager
- }
- fileName: {
- type: 'String'
- defaultValue: 'settings.json'
- }
- folderPath: {
- type: 'String'
- defaultValue: configContainerName
- }
- }
- }
-}
-
-//------------------------------------------------------------------------------
-// msexports_ExecuteETL pipeline
-// Triggered by msexports_ManifestAdded trigger
-//------------------------------------------------------------------------------
-@description('Queues the msexports_ETL_ingestion pipeline.')
-resource pipeline_ExecuteExportsETL 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = {
- name: '${safeExportContainerName}_ExecuteETL'
- parent: dataFactory
- properties: {
- activities: [
- { // Wait
- name: 'Wait'
- description: 'Files may not be available immediately after being created.'
- type: 'Wait'
- dependsOn: []
- userProperties: []
- typeProperties: {
- waitTimeInSeconds: 60
- }
- }
- { // Read Manifest
- name: 'Read Manifest'
- description: 'Load the export manifest to determine the scope, dataset, and date range.'
- type: 'Lookup'
- dependsOn: [
- {
- activity: 'Wait'
- dependencyConditions: ['Completed']
- }
- ]
- policy: {
- timeout: '0.12:00:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- source: {
- type: 'JsonSource'
- storeSettings: {
- type: 'AzureBlobFSReadSettings'
- recursive: true
- enablePartitionDiscovery: false
- }
- formatSettings: {
- type: 'JsonReadSettings'
- }
- }
- dataset: {
- referenceName: dataset_manifest.name
- type: 'DatasetReference'
- parameters: {
- fileName: {
- value: '@pipeline().parameters.fileName'
- type: 'Expression'
- }
- folderPath: {
- value: '@pipeline().parameters.folderPath'
- type: 'Expression'
- }
- }
- }
- }
- }
- { // Set Has No Rows
- name: 'Set Has No Rows'
- description: 'Check the row count '
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Read Manifest'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'hasNoRows'
- value: {
- value: '@or(equals(activity(\'Read Manifest\').output.firstRow.blobCount, null), equals(activity(\'Read Manifest\').output.firstRow.blobCount, 0))'
- type: 'Expression'
- }
- }
- }
- { // Set Export Dataset Type
- name: 'Set Export Dataset Type'
- description: 'Save the dataset type from the export manifest.'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Read Manifest'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'exportDatasetType'
- value: {
- value: '@activity(\'Read Manifest\').output.firstRow.exportConfig.type'
- type: 'Expression'
- }
- }
- }
- { // Set MCA Column
- name: 'Set MCA Column'
- description: 'Determines if the dataset schema has channel-specific columns and saves the column name that only exists in MCA to determine if it is an MCA dataset.'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Set Export Dataset Type'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'mcaColumnToCheck'
- value: {
- // cSpell:ignore pricesheet, reservationtransactions, reservationrecommendations
- value: '@if(contains(createArray(\'pricesheet\', \'reservationtransactions\'), toLower(variables(\'exportDatasetType\'))), \'BillingProfileId\', if(equals(toLower(variables(\'exportDatasetType\')), \'reservationrecommendations\'), \'Net Savings\', null))'
- type: 'Expression'
- }
- }
- }
- { // Set Export Dataset Version
- name: 'Set Export Dataset Version'
- description: 'Save the dataset version from the export manifest.'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Read Manifest'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'exportDatasetVersion'
- value: {
- value: '@activity(\'Read Manifest\').output.firstRow.exportConfig.dataVersion'
- type: 'Expression'
- }
- }
- }
- { // Detect Channel
- name: 'Detect Channel'
- description: 'Determines what channel this export is from. Switch statement handles the different file types if the mcaColumnToCheck variable is set.'
- type: 'Switch'
- dependsOn: [
- {
- activity: 'Set Has No Rows'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- {
- activity: 'Set MCA Column'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- {
- activity: 'Set Export Dataset Version'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- on: {
- value: '@if(or(empty(variables(\'mcaColumnToCheck\')), variables(\'hasNoRows\')), \'ignore\', last(array(split(activity(\'Read Manifest\').output.firstRow.blobs[0].blobName, \'.\'))))'
- type: 'Expression'
- }
- cases: [
- { // csv
- value: 'csv'
- activities: [
- {
- name: 'Check for MCA Column in CSV'
- description: 'Checks the dataset to determine if the applicable MCA-specific column exists.'
- type: 'Lookup'
- dependsOn: []
- policy: {
- timeout: '0.12:00:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- source: {
- type: 'DelimitedTextSource'
- storeSettings: {
- type: 'AzureBlobFSReadSettings'
- recursive: false
- enablePartitionDiscovery: false
- }
- formatSettings: {
- type: 'DelimitedTextReadSettings'
- }
- }
- dataset: {
- referenceName: dataset_msexports.name
- type: 'DatasetReference'
- parameters: {
- blobPath: {
- value: '@activity(\'Read Manifest\').output.firstRow.blobs[0].blobName'
- type: 'Expression'
- }
- }
- }
- }
- }
- {
- name: 'Set Schema File with Channel in CSV'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Check for MCA Column in CSV'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'schemaFile'
- value: {
- value: '@toLower(concat(variables(\'exportDatasetType\'), \'_\', variables(\'exportDatasetVersion\'), if(and(contains(activity(\'Check for MCA Column in CSV\').output, \'firstRow\'), contains(activity(\'Check for MCA Column in CSV\').output.firstRow, variables(\'mcaColumnToCheck\'))), \'_mca\', \'_ea\'), \'.json\'))'
- type: 'Expression'
- }
- }
- }
- ]
- }
- { // gz
- value: 'gz'
- activities: [
- {
- name: 'Check for MCA Column in Gzip CSV'
- description: 'Checks the dataset to determine if the applicable MCA-specific column exists.'
- type: 'Lookup'
- dependsOn: []
- policy: {
- timeout: '0.12:00:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- source: {
- type: 'DelimitedTextSource'
- storeSettings: {
- type: 'AzureBlobFSReadSettings'
- recursive: false
- enablePartitionDiscovery: false
- }
- formatSettings: {
- type: 'DelimitedTextReadSettings'
- }
- }
- dataset: {
- referenceName: dataset_msexports_gzip.name
- type: 'DatasetReference'
- parameters: {
- blobPath: {
- value: '@activity(\'Read Manifest\').output.firstRow.blobs[0].blobName'
- type: 'Expression'
- }
- }
- }
- }
- }
- {
- name: 'Set Schema File with Channel in Gzip CSV'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Check for MCA Column in Gzip CSV'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'schemaFile'
- value: {
- value: '@toLower(concat(variables(\'exportDatasetType\'), \'_\', variables(\'exportDatasetVersion\'), if(and(contains(activity(\'Check for MCA Column in Gzip CSV\').output, \'firstRow\'), contains(activity(\'Check for MCA Column in Gzip CSV\').output.firstRow, variables(\'mcaColumnToCheck\'))), \'_mca\', \'_ea\'), \'.json\'))'
- type: 'Expression'
- }
- }
- }
- ]
- }
- { // parquet
- value: 'parquet'
- activities: [
- {
- name: 'Check for MCA Column in Parquet'
- description: 'Checks the dataset to determine if the applicable MCA-specific column exists.'
- type: 'Lookup'
- dependsOn: []
- policy: {
- timeout: '0.12:00:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- source: {
- type: 'ParquetSource'
- storeSettings: {
- type: 'AzureBlobFSReadSettings'
- recursive: false
- enablePartitionDiscovery: false
- }
- formatSettings: {
- type: 'ParquetReadSettings'
- }
- }
- dataset: {
- referenceName: dataset_msexports_parquet.name
- type: 'DatasetReference'
- parameters: {
- blobPath: {
- value: '@activity(\'Read Manifest\').output.firstRow.blobs[0].blobName'
- type: 'Expression'
- }
- }
- }
- }
- }
- {
- name: 'Set Schema File with Channel for Parquet'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Check for MCA Column in Parquet'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'schemaFile'
- value: {
- value: '@toLower(concat(variables(\'exportDatasetType\'), \'_\', variables(\'exportDatasetVersion\'), if(and(contains(activity(\'Check for MCA Column in Parquet\').output, \'firstRow\'), contains(activity(\'Check for MCA Column in Parquet\').output.firstRow, variables(\'mcaColumnToCheck\'))), \'_mca\', \'_ea\'), \'.json\'))'
- type: 'Expression'
- }
- }
- }
- ]
- }
- ]
- defaultActivities: [
- {
- name: 'Set Schema File'
- type: 'SetVariable'
- dependsOn: []
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'schemaFile'
- value: {
- value: '@toLower(concat(variables(\'exportDatasetType\'), \'_\', variables(\'exportDatasetVersion\'), \'.json\'))'
- type: 'Expression'
- }
- }
- }
- ]
- }
- }
- { // Set Scope
- name: 'Set Scope'
- description: 'Save the scope from the export manifest.'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Read Manifest'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'scope'
- value: {
- value: '@split(toLower(activity(\'Read Manifest\').output.firstRow.exportConfig.resourceId), \'/providers/microsoft.costmanagement/exports/\')[0]'
- type: 'Expression'
- }
- }
- }
- { // Set Date
- name: 'Set Date'
- description: 'Save the exported month from the export manifest.'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Read Manifest'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'date'
- value: {
- value: '@replace(substring(activity(\'Read Manifest\').output.firstRow.runInfo.startDate, 0, 7), \'-\', \'\')'
- type: 'Expression'
- }
- }
- }
- { // Error: ManifestReadFailed
- name: 'Failed to Read Manifest'
- type: 'Fail'
- dependsOn: [
- {
- activity: 'Set Date'
- dependencyConditions: ['Failed']
- }
- {
- activity: 'Set Export Dataset Type'
- dependencyConditions: ['Failed']
- }
- {
- activity: 'Set Scope'
- dependencyConditions: ['Failed']
- }
- {
- activity: 'Read Manifest'
- dependencyConditions: ['Failed']
- }
- {
- activity: 'Set Export Dataset Version'
- dependencyConditions: ['Failed']
- }
- {
- activity: 'Detect Channel'
- dependencyConditions: ['Failed']
- }
- ]
- userProperties: []
- typeProperties: {
- message: {
- value: '@concat(\'Failed to read the manifest file for this export run. Manifest path: \', pipeline().parameters.folderPath)'
- type: 'Expression'
- }
- errorCode: 'ManifestReadFailed'
- }
- }
- { // Check Schema
- name: 'Check Schema'
- description: 'Verify that the schema file exists in storage.'
- type: 'GetMetadata'
- dependsOn: [
- {
- activity: 'Set Scope'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- {
- activity: 'Set Date'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- {
- activity: 'Detect Channel'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- timeout: '0.12:00:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- dataset: {
- referenceName: dataset_config.name
- type: 'DatasetReference'
- parameters: {
- fileName: {
- value: '@variables(\'schemaFile\')'
- type: 'Expression'
- }
- folderPath: '${configContainerName}/schemas'
- }
- }
- fieldList: ['exists']
- storeSettings: {
- type: 'AzureBlobFSReadSettings'
- recursive: true
- enablePartitionDiscovery: false
- }
- formatSettings: {
- type: 'JsonReadSettings'
- }
- }
- }
- { // Error: SchemaNotFound
- name: 'Schema Not Found'
- type: 'Fail'
- dependsOn: [
- {
- activity: 'Check Schema'
- dependencyConditions: ['Failed']
- }
- ]
- userProperties: []
- typeProperties: {
- message: {
- value: '@concat(\'The \', variables(\'schemaFile\'), \' schema mapping file was not found. Please confirm version \', variables(\'exportDatasetVersion\'), \' of the \', variables(\'exportDatasetType\'), \' dataset is supported by this version of FinOps hubs. You may need to upgrade to a newer release. To add support for another dataset, you can create a custom mapping file.\')'
- type: 'Expression'
- }
- errorCode: 'SchemaNotFound'
- }
- }
- { // Set Hub Dataset
- name: 'Set Hub Dataset'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Set Export Dataset Type'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'hubDataset'
- value: {
- value: '@if(equals(toLower(variables(\'exportDatasetType\')), \'focuscost\'), \'Costs\', if(equals(toLower(variables(\'exportDatasetType\')), \'pricesheet\'), \'Prices\', if(equals(toLower(variables(\'exportDatasetType\')), \'reservationdetails\'), \'CommitmentDiscountUsage\', if(equals(toLower(variables(\'exportDatasetType\')), \'reservationrecommendations\'), \'Recommendations\', if(equals(toLower(variables(\'exportDatasetType\')), \'reservationtransactions\'), \'Transactions\', if(equals(toLower(variables(\'exportDatasetType\')), \'actualcost\'), \'ActualCosts\', if(equals(toLower(variables(\'exportDatasetType\')), \'amortizedcost\'), \'AmortizedCosts\', toLower(variables(\'exportDatasetType\')))))))))'
- type: 'Expression'
- }
- }
- }
- { // Set Destination Folder
- name: 'Set Destination Folder'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Check Schema'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- {
- activity: 'Set Hub Dataset'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'destinationFolder'
- value: {
- value: '@replace(concat(variables(\'hubDataset\'),\'/\',substring(variables(\'date\'), 0, 4),\'/\',substring(variables(\'date\'), 4, 2),\'/\',toLower(variables(\'scope\')), if(equals(variables(\'hubDataset\'), \'Recommendations\'), activity(\'Read Manifest\').output.firstRow.exportConfig.exportName, \'\')),\'//\',\'/\')'
- type: 'Expression'
- }
- }
- }
- { // For Each Blob
- name: 'For Each Blob'
- description: 'Loop thru each exported file listed in the manifest.'
- type: 'ForEach'
- dependsOn: [
- {
- activity: 'Set Destination Folder'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- items: {
- value: '@if(variables(\'hasNoRows\'), json(\'[]\'), activity(\'Read Manifest\').output.firstRow.blobs)'
- type: 'Expression'
- }
- batchCount: enablePublicAccess ? 30 : 4 // so we don't overload the managed runtime
- isSequential: false
- activities: [
- { // Execute
- name: 'Execute'
- description: 'Run the ingestion ETL pipeline.'
- type: 'ExecutePipeline'
- dependsOn: []
- policy: {
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- pipeline: {
- referenceName: pipeline_ToIngestion.name
- type: 'PipelineReference'
- }
- waitOnCompletion: true
- parameters: {
- blobPath: {
- value: '@item().blobName'
- type: 'Expression'
- }
- destinationFolder: {
- value: '@variables(\'destinationFolder\')'
- type: 'Expression'
- }
- destinationFile: {
- value: '@last(array(split(replace(replace(item().blobName, \'.gz\', \'\'), \'.csv\', \'.parquet\'), \'/\')))'
- type: 'Expression'
- }
- ingestionId: {
- value: '@activity(\'Read Manifest\').output.firstRow.runInfo.runId'
- type: 'Expression'
- }
- schemaFile: {
- value: '@variables(\'schemaFile\')'
- type: 'Expression'
- }
- exportDatasetType: {
- value: '@variables(\'exportDatasetType\')'
- type: 'Expression'
- }
- exportDatasetVersion: {
- value: '@variables(\'exportDatasetVersion\')'
- type: 'Expression'
- }
- }
- }
- }
- ]
- }
- }
- { // Copy Manifest
- name: 'Copy Manifest'
- description: 'Copy the manifest to the ingestion container to trigger ADX ingestion'
- type: 'Copy'
- dependsOn: [
- {
- activity: 'For Each Blob'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- timeout: '0.12:00:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- source: {
- type: 'JsonSource'
- storeSettings: {
- type: 'AzureBlobFSReadSettings'
- recursive: true
- enablePartitionDiscovery: false
- }
- formatSettings: {
- type: 'JsonReadSettings'
- }
- }
- sink: {
- type: 'JsonSink'
- storeSettings: {
- type: 'AzureBlobFSWriteSettings'
- }
- formatSettings: {
- type: 'JsonWriteSettings'
- }
- }
- enableStaging: false
- }
- inputs: [
- {
- referenceName: dataset_manifest.name
- type: 'DatasetReference'
- parameters: {
- fileName: 'manifest.json'
- folderPath: {
- value: '@pipeline().parameters.folderPath'
- type: 'Expression'
- }
- }
- }
- ]
- outputs: [
- {
- referenceName: dataset_manifest.name
- type: 'DatasetReference'
- parameters: {
- fileName: 'manifest.json'
- folderPath: {
- value: '@concat(\'${ingestionContainerName}/\', variables(\'destinationFolder\'))'
- type: 'Expression'
- }
- }
- }
- ]
- }
- ]
- parameters: {
- folderPath: {
- type: 'string'
- }
- fileName: {
- type: 'string'
- }
- }
- variables: {
- date: {
- type: 'String'
- }
- destinationFolder: {
- type: 'String'
- }
- exportDatasetType: {
- type: 'String'
- }
- exportDatasetVersion: {
- type: 'String'
- }
- hasNoRows: {
- type: 'Boolean'
- }
- hubDataset: {
- type: 'String'
- }
- mcaColumnToCheck: {
- type: 'String'
- }
- schemaFile: {
- type: 'String'
- }
- scope: {
- type: 'String'
- }
- }
- annotations: [
- 'New export'
- ]
- }
-}
-
-//------------------------------------------------------------------------------
-// msexports_ETL_ingestion pipeline
-// Triggered by msexports_ExecuteETL
-//------------------------------------------------------------------------------
-@description('Transforms CSV data to a standard schema and converts to Parquet.')
-resource pipeline_ToIngestion 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = {
- name: '${safeExportContainerName}_ETL_${safeIngestionContainerName}'
- parent: dataFactory
- properties: {
- activities: [
- { // Get Existing Parquet Files
- name: 'Get Existing Parquet Files'
- description: 'Get the previously ingested files so we can remove any older data. This is necessary to avoid data duplication in reports.'
- type: 'GetMetadata'
- dependsOn: []
- policy: {
- timeout: '0.12:00:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- dataset: {
- referenceName: dataset_ingestion_files.name
- type: 'DatasetReference'
- parameters: {
- folderPath: '@pipeline().parameters.destinationFolder'
- }
- }
- fieldList: [
- 'childItems'
- ]
- storeSettings: {
- type: 'AzureBlobFSReadSettings'
- enablePartitionDiscovery: false
- }
- formatSettings: {
- type: 'ParquetReadSettings'
- }
- }
- }
- { // Filter Out Current Exports
- name: 'Filter Out Current Exports'
- description: 'Remove existing files from the current export so those files do not get deleted.'
- type: 'Filter'
- dependsOn: [
- {
- activity: 'Get Existing Parquet Files'
- dependencyConditions: [
- 'Completed'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- items: {
- value: '@if(contains(activity(\'Get Existing Parquet Files\').output, \'childItems\'), activity(\'Get Existing Parquet Files\').output.childItems, json(\'[]\'))'
- type: 'Expression'
- }
- condition: {
- // cSpell:ignore endswith
- value: '@and(endswith(item().name, \'.parquet\'), not(startswith(item().name, concat(pipeline().parameters.ingestionId, \'${ingestionIdFileNameSeparator}\'))))'
- type: 'Expression'
- }
- }
- }
- { // Load Schema Mappings
- name: 'Load Schema Mappings'
- description: 'Get schema mapping file to use for the CSV to parquet conversion.'
- type: 'Lookup'
- dependsOn: []
- policy: {
- timeout: '0.12:00:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- source: {
- type: 'JsonSource'
- storeSettings: {
- type: 'AzureBlobFSReadSettings'
- recursive: true
- enablePartitionDiscovery: false
- }
- formatSettings: {
- type: 'JsonReadSettings'
- }
- }
- dataset: {
- referenceName: dataset_config.name
- type: 'DatasetReference'
- parameters: {
- fileName: {
- value: '@toLower(pipeline().parameters.schemaFile)'
- type: 'Expression'
- }
- folderPath: '${configContainerName}/schemas'
- }
- }
- }
- }
- { // Error: SchemaLoadFailed
- name: 'Failed to Load Schema'
- type: 'Fail'
- dependsOn: [
- {
- activity: 'Load Schema Mappings'
- dependencyConditions: [
- 'Failed'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- message: {
- value: '@concat(\'Unable to load the \', pipeline().parameters.schemaFile, \' schema file. Please confirm the schema and version are supported for FinOps hubs ingestion. Unsupported files will remain in the msexports container.\')'
- type: 'Expression'
- }
- errorCode: 'SchemaLoadFailed'
- }
- }
- { // Set Additional Columns
- name: 'Set Additional Columns'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Load Schema Mappings'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'additionalColumns'
- value: {
- value: '@intersection(array(json(concat(\'[{"name":"x_SourceProvider","value":"Microsoft"},{"name":"x_SourceName","value":"Cost Management"},{"name":"x_SourceType","value":"\', pipeline().parameters.exportDatasetVersion, \'"},{"name":"x_SourceVersion","value":"\', pipeline().parameters.exportDatasetVersion, \'"}\'))), activity(\'Load Schema Mappings\').output.firstRow.additionalColumns)'
- type: 'Expression'
- }
- }
- }
- { // For Each Old File
- name: 'For Each Old File'
- description: 'Loop thru each of the existing files from previous exports.'
- type: 'ForEach'
- dependsOn: [
- {
- activity: 'Convert to Parquet'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- {
- activity: 'Filter Out Current Exports'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- items: {
- value: '@activity(\'Filter Out Current Exports\').output.Value'
- type: 'Expression'
- }
- activities: [
- { // Delete Old Ingested File
- name: 'Delete Old Ingested File'
- description: 'Delete the previously ingested files from older exports.'
- type: 'Delete'
- dependsOn: []
- policy: {
- timeout: '0.12:00:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- dataset: {
- referenceName: dataset_ingestion.name
- type: 'DatasetReference'
- parameters: {
- blobPath: {
- value: '@concat(pipeline().parameters.destinationFolder, \'/\', item().name)'
- type: 'Expression'
- }
- }
- }
- enableLogging: false
- storeSettings: {
- type: 'AzureBlobFSReadSettings'
- recursive: false
- enablePartitionDiscovery: false
- }
- }
- }
- ]
- }
- }
- { // Set Destination Path
- name: 'Set Destination Path'
- type: 'SetVariable'
- dependsOn: []
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'destinationPath'
- value: {
- value: '@concat(pipeline().parameters.destinationFolder, \'/\', pipeline().parameters.ingestionId, \'${ingestionIdFileNameSeparator}\', pipeline().parameters.destinationFile)'
- type: 'Expression'
- }
- }
- }
- { // Convert to Parquet
- name: 'Convert to Parquet'
- description: 'Convert CSV to parquet and move the file to the ${ingestionContainerName} container.'
- type: 'Switch'
- dependsOn: [
- {
- activity: 'Set Destination Path'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- {
- activity: 'Load Schema Mappings'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- {
- activity: 'Set Additional Columns'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- on: {
- value: '@last(array(split(pipeline().parameters.blobPath, \'.\')))'
- type: 'Expression'
- }
- cases: [
- { // CSV
- value: 'csv'
- activities: [
- { // Convert CSV File
- name: 'Convert CSV File'
- type: 'Copy'
- dependsOn: []
- policy: {
- timeout: '0.00:10:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- source: {
- type: 'DelimitedTextSource'
- additionalColumns: {
- value: '@variables(\'additionalColumns\')'
- type: 'Expression'
- }
- storeSettings: {
- type: 'AzureBlobFSReadSettings'
- recursive: true
- enablePartitionDiscovery: false
- }
- formatSettings: {
- type: 'DelimitedTextReadSettings'
- }
- }
- sink: {
- type: 'ParquetSink'
- storeSettings: {
- type: 'AzureBlobFSWriteSettings'
- }
- formatSettings: {
- type: 'ParquetWriteSettings'
- fileExtension: '.parquet'
- }
- }
- enableStaging: false
- parallelCopies: 1
- validateDataConsistency: false
- translator: {
- value: '@activity(\'Load Schema Mappings\').output.firstRow.translator'
- type: 'Expression'
- }
- }
- inputs: [
- {
- referenceName: dataset_msexports.name
- type: 'DatasetReference'
- parameters: {
- blobPath: {
- value: '@pipeline().parameters.blobPath'
- type: 'Expression'
- }
- }
- }
- ]
- outputs: [
- {
- referenceName: dataset_ingestion.name
- type: 'DatasetReference'
- parameters: {
- blobPath: {
- value: '@variables(\'destinationPath\')'
- type: 'Expression'
- }
- }
- }
- ]
- }
- ]
- }
- { // GZ
- value: 'gz'
- activities: [
- { // Convert GZip CSV File
- name: 'Convert GZip CSV File'
- type: 'Copy'
- dependsOn: []
- policy: {
- timeout: '0.00:10:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- source: {
- type: 'DelimitedTextSource'
- additionalColumns: {
- value: '@variables(\'additionalColumns\')'
- type: 'Expression'
- }
- storeSettings: {
- type: 'AzureBlobFSReadSettings'
- recursive: true
- enablePartitionDiscovery: false
- }
- formatSettings: {
- type: 'DelimitedTextReadSettings'
- }
- }
- sink: {
- type: 'ParquetSink'
- storeSettings: {
- type: 'AzureBlobFSWriteSettings'
- }
- formatSettings: {
- type: 'ParquetWriteSettings'
- fileExtension: '.parquet'
- }
- }
- enableStaging: false
- parallelCopies: 1
- validateDataConsistency: false
- translator: {
- value: '@activity(\'Load Schema Mappings\').output.firstRow.translator'
- type: 'Expression'
- }
- }
- inputs: [
- {
- referenceName: dataset_msexports_gzip.name
- type: 'DatasetReference'
- parameters: {
- blobPath: {
- value: '@pipeline().parameters.blobPath'
- type: 'Expression'
- }
- }
- }
- ]
- outputs: [
- {
- referenceName: dataset_ingestion.name
- type: 'DatasetReference'
- parameters: {
- blobPath: {
- value: '@variables(\'destinationPath\')'
- type: 'Expression'
- }
- }
- }
- ]
- }
- ]
- }
- { // Parquet
- value: 'parquet'
- activities: [
- { // Move Parquet File
- name: 'Move Parquet File'
- type: 'Copy'
- dependsOn: []
- policy: {
- timeout: '0.00:05:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- source: {
- type: 'ParquetSource'
- additionalColumns: {
- value: '@variables(\'additionalColumns\')'
- type: 'Expression'
- }
- storeSettings: {
- type: 'AzureBlobFSReadSettings'
- recursive: true
- enablePartitionDiscovery: false
- }
- formatSettings: {
- type: 'ParquetReadSettings'
- }
- }
- sink: {
- type: 'ParquetSink'
- storeSettings: {
- type: 'AzureBlobFSWriteSettings'
- }
- formatSettings: {
- type: 'ParquetWriteSettings'
- fileExtension: '.parquet'
- }
- }
- enableStaging: false
- parallelCopies: 1
- validateDataConsistency: false
- }
- inputs: [
- {
- referenceName: dataset_msexports_parquet.name
- type: 'DatasetReference'
- parameters: {
- blobPath: {
- value: '@pipeline().parameters.blobPath'
- type: 'Expression'
- }
- }
- }
- ]
- outputs: [
- {
- referenceName: dataset_ingestion.name
- type: 'DatasetReference'
- parameters: {
- blobPath: {
- value: '@variables(\'destinationPath\')'
- type: 'Expression'
- }
- }
- }
- ]
- }
- ]
- }
- ]
- defaultActivities: [
- { // Error: UnsupportedFileType
- name: 'Unsupported File Type'
- type: 'Fail'
- dependsOn: []
- userProperties: []
- typeProperties: {
- message: {
- value: '@concat(\'Unable to ingest the specified export file because the file type is not supported. File: \', pipeline().parameters.blobPath)'
- type: 'Expression'
- }
- errorCode: 'UnsupportedExportFileType'
- }
- }
- ]
- }
- }
- { // Read Hub Config
- name: 'Read Hub Config'
- description: 'Read the hub config to determine if the export should be retained.'
- type: 'Lookup'
- dependsOn: []
- policy: {
- timeout: '0.12:00:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- source: {
- type: 'JsonSource'
- storeSettings: {
- type: 'AzureBlobFSReadSettings'
- recursive: false
- enablePartitionDiscovery: false
- }
- formatSettings: {
- type: 'JsonReadSettings'
- }
- }
- dataset: {
- referenceName: dataset_config.name
- type: 'DatasetReference'
- parameters: {
- fileName: 'settings.json'
- folderPath: configContainerName
- }
- }
- }
- }
- { // If Not Retaining Exports
- name: 'If Not Retaining Exports'
- description: 'If the msexports retention period <= 0, delete the source file. The main reason to keep the source file is to allow for troubleshooting and reprocessing in the future.'
- type: 'IfCondition'
- dependsOn: [
- {
- activity: 'Convert to Parquet'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- {
- activity: 'Read Hub Config'
- dependencyConditions: [
- 'Completed'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- expression: {
- value: '@lessOrEquals(coalesce(activity(\'Read Hub Config\').output.firstRow.retention.msexports.days, 0), 0)'
- type: 'Expression'
- }
- ifTrueActivities: [
- { // Delete Source File
- name: 'Delete Source File'
- description: 'Delete the exported data file to keep storage costs down. This file is not referenced by any reporting systems.'
- type: 'Delete'
- dependsOn: []
- policy: {
- timeout: '0.12:00:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- dataset: {
- referenceName: dataset_msexports_parquet.name
- type: 'DatasetReference'
- parameters: {
- blobPath: {
- value: '@pipeline().parameters.blobPath'
- type: 'Expression'
- }
- }
- }
- enableLogging: false
- storeSettings: {
- type: 'AzureBlobFSReadSettings'
- recursive: true
- enablePartitionDiscovery: false
- }
- }
- }
- ]
- }
- }
- ]
- parameters: {
- blobPath: {
- type: 'String'
- }
- destinationFile: {
- type: 'string'
- }
- destinationFolder: {
- type: 'string'
- }
- ingestionId: {
- type: 'string'
- }
- schemaFile: {
- type: 'string'
- }
- exportDatasetType: {
- type: 'string'
- }
- exportDatasetVersion: {
- type: 'string'
- }
- }
- variables: {
- additionalColumns: {
- type: 'Array'
- }
- destinationPath: {
- type: 'String'
- }
- }
- annotations: []
- }
-}
-
-//------------------------------------------------------------------------------
-// ingestion_ETL_dataExplorer pipeline
-// Triggered by ingestion_ExecuteETL
-//------------------------------------------------------------------------------
-@description('Ingests parquet data into an Azure Data Explorer cluster.')
-resource pipeline_ToDataExplorer 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = if (deployDataExplorer || useFabric) {
- name: '${safeIngestionContainerName}_ETL_dataExplorer'
- parent: dataFactory
- properties: {
- activities: [
- { // Read Hub Config
- name: 'Read Hub Config'
- description: 'Read the hub config to determine how long data should be retained.'
- type: 'Lookup'
- dependsOn: [
- ]
- policy: {
- timeout: '0.12:00:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- source: {
- type: 'JsonSource'
- storeSettings: {
- type: 'AzureBlobFSReadSettings'
- recursive: false
- enablePartitionDiscovery: false
- }
- formatSettings: {
- type: 'JsonReadSettings'
- }
- }
- dataset: {
- referenceName: dataset_config.name
- type: 'DatasetReference'
- parameters: {
- fileName: 'settings.json'
- folderPath: configContainerName
- }
- }
- }
- }
- { // Set Final Retention Months
- name: 'Set Final Retention Months'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Read Hub Config'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'finalRetentionMonths'
- value: {
- value: '@coalesce(activity(\'Read Hub Config\').output.firstRow.retention.final.months, 999)'
- type: 'Expression'
- }
- }
- }
- { // Until Capacity Is Available
- name: 'Until Capacity Is Available'
- type: 'Until'
- dependsOn: [
- {
- activity: 'Set Final Retention Months'
- dependencyConditions: [
- 'Completed'
- 'Skipped'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- expression: {
- value: '@equals(variables(\'tryAgain\'), false)'
- type: 'Expression'
- }
- activities: [
- { // Confirm Ingestion Capacity
- name: 'Confirm Ingestion Capacity'
- type: 'AzureDataExplorerCommand'
- dependsOn: []
- policy: {
- timeout: '0.12:00:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- command: '.show capacity | where Resource == \'Ingestions\' | project Remaining'
- commandTimeout: '00:20:00'
- }
- linkedServiceName: {
- referenceName: linkedService_dataExplorer.name
- type: 'LinkedServiceReference'
- }
- }
- { // If Has Capacity
- name: 'If Has Capacity'
- type: 'IfCondition'
- dependsOn: [
- {
- activity: 'Confirm Ingestion Capacity'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- expression: {
- value: '@or(equals(activity(\'Confirm Ingestion Capacity\').output.count, 0), greater(activity(\'Confirm Ingestion Capacity\').output.value[0].Remaining, 0))'
- type: 'Expression'
- }
- ifFalseActivities: [
- { // Wait for Ingestion
- name: 'Wait for Ingestion'
- type: 'Wait'
- dependsOn: []
- userProperties: []
- typeProperties: {
- waitTimeInSeconds: 15
- }
- }
- { // Try Again
- name: 'Try Again'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Wait for Ingestion'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'tryAgain'
- value: true
- }
- }
- ]
- ifTrueActivities: [
- { // Pre-Ingest Cleanup
- name: 'Pre-Ingest Cleanup'
- description: 'Cost Management exports include all month-to-date data from the previous export run. To ensure data is not double-reported, it must be dropped from the raw table before ingestion completes. Remove previous ingestions into the raw table for the month and any previous runs of the current ingestion month file in any table.'
- type: 'AzureDataExplorerCommand'
- dependsOn: []
- policy: {
- timeout: '0.12:00:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- typeProperties: {
- command: {
- value: '@concat(\'.drop extents <| .show extents | where (TableName == "\', pipeline().parameters.table, \'" and Tags !has "drop-by:\', pipeline().parameters.ingestionId, \'" and Tags has "drop-by:\', pipeline().parameters.folderPath, \'") or (Tags has "drop-by:\', pipeline().parameters.ingestionId, \'" and Tags has "drop-by:\', pipeline().parameters.folderPath, \'/\', pipeline().parameters.originalFileName, \'")\')'
- type: 'Expression'
- }
- commandTimeout: '00:20:00'
- }
- linkedServiceName: {
- referenceName: linkedService_dataExplorer.name
- type: 'LinkedServiceReference'
- parameters: {
- database: dataExplorerIngestionDatabase
- }
- }
- }
- { // Ingest Data
- name: 'Ingest Data'
- type: 'AzureDataExplorerCommand'
- dependsOn: [
- {
- activity: 'Pre-Ingest Cleanup'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- timeout: '0.12:00:00'
- retry: 3
- retryIntervalInSeconds: 120
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- command: {
- // cSpell:ignore abfss, toscalar
- value: '@concat(\'.ingest into table \', pipeline().parameters.table, \' ("abfss://${ingestionContainerName}@${storageAccount.name}.dfs.${environment().suffixes.storage}/\', pipeline().parameters.folderPath, \'/\', pipeline().parameters.fileName, \';${useFabric ? 'impersonate' : 'managed_identity=system'}") with (format="parquet", ingestionMappingReference="\', pipeline().parameters.table, \'_mapping", tags="[\\"drop-by:\', pipeline().parameters.ingestionId, \'\\", \\"drop-by:\', pipeline().parameters.folderPath, \'/\', pipeline().parameters.originalFileName, \'\\", \\"drop-by:ftk-version-${ftkVersion}\\"]"); print Success = assert(iff(toscalar($command_results | project-keep HasErrors) == false, true, false), "Ingestion Failed")\')'
- type: 'Expression'
- }
- commandTimeout: '01:00:00'
- }
- linkedServiceName: {
- referenceName: linkedService_dataExplorer.name
- type: 'LinkedServiceReference'
- parameters: {
- database: dataExplorerIngestionDatabase
- }
- }
- }
- { // Post-Ingest Cleanup
- name: 'Post-Ingest Cleanup'
- description: 'Cost Management exports include all month-to-date data from the previous export run. To ensure data is not double-reported, it must be dropped after ingestion completes. Remove the current ingestion month file from raw and any old ingestions for the month from the final table.'
- type: 'AzureDataExplorerCommand'
- dependsOn: [
- {
- activity: 'Ingest Data'
- dependencyConditions: [
- 'Completed'
- ]
- }
- ]
- policy: {
- timeout: '0.12:00:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- typeProperties: {
- command: {
- // cSpell:ignore startofmonth, strcat, todatetime
- value: '@concat(\'.drop extents <| .show extents | extend isOldFinalData = (TableName startswith "\', replace(pipeline().parameters.table, \'_raw\', \'_final_v\'), \'" and Tags !has "drop-by:\', pipeline().parameters.ingestionId, \'" and Tags has "drop-by:\', pipeline().parameters.folderPath, \'") | extend isPastFinalRetention = (TableName startswith "\', replace(pipeline().parameters.table, \'_raw\', \'_final_v\'), \'" and todatetime(substring(strcat(replace_string(extract("drop-by:[A-Za-z]+/(\\\\d{4}/\\\\d{2}(/\\\\d{2})?)", 1, Tags), "/", "-"), "-01"), 0, 10)) < datetime_add("month", -\', if(lessOrEquals(variables(\'finalRetentionMonths\'), 0), 0, variables(\'finalRetentionMonths\')), \', startofmonth(now()))) | where isOldFinalData or isPastFinalRetention\')'
- type: 'Expression'
- }
- commandTimeout: '00:20:00'
- }
- linkedServiceName: {
- referenceName: linkedService_dataExplorer.name
- type: 'LinkedServiceReference'
- parameters: {
- database: dataExplorerIngestionDatabase
- }
- }
- }
- { // Ingestion Complete
- name: 'Ingestion Complete'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Post-Ingest Cleanup'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'tryAgain'
- value: false
- }
- }
- { // Abort On Ingestion Error
- name: 'Abort On Ingestion Error'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Ingest Data'
- dependencyConditions: [
- 'Failed'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'tryAgain'
- value: false
- }
- }
- { // Error: DataExplorerIngestionFailed
- name: 'Ingestion Failed Error'
- type: 'Fail'
- dependsOn: [
- {
- activity: 'Abort On Ingestion Error'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- message: {
- value: '@concat(\'Data Explorer ingestion into the \', pipeline().parameters.table, \' table failed. Please fix the error and rerun ingestion for the following folder path: "\', pipeline().parameters.folderPath, \'". File: \', pipeline().parameters.originalFileName, \'. Error: \', if(greater(length(activity(\'Ingest Data\').output.errors), 0), activity(\'Ingest Data\').output.errors[0].Message, \'Unknown\'), \' (Code: \', if(greater(length(activity(\'Ingest Data\').output.errors), 0), activity(\'Ingest Data\').output.errors[0].Code, \'None\'), \')\')'
- type: 'Expression'
- }
- errorCode: 'DataExplorerIngestionFailed'
- }
- }
- { // Abort On Pre-Ingest Drop Error
- name: 'Abort On Pre-Ingest Drop Error'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Pre-Ingest Cleanup'
- dependencyConditions: [
- 'Failed'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'tryAgain'
- value: false
- }
- }
- { // Error: DataExplorerPreIngestionDropFailed
- name: 'Pre-Ingest Drop Failed Error'
- type: 'Fail'
- dependsOn: [
- {
- activity: 'Abort On Pre-Ingest Drop Error'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- message: {
- value: '@concat(\'Data Explorer pre-ingestion cleanup (drop extents from raw table) for the \', pipeline().parameters.table, \' table failed. Ingestion was not completed. Please fix the error and rerun ingestion for the following folder path: "\', pipeline().parameters.folderPath, \'". File: \', pipeline().parameters.originalFileName, \'. Error: \', if(greater(length(activity(\'Pre-Ingest Cleanup\').output.errors), 0), activity(\'Pre-Ingest Cleanup\').output.errors[0].Message, \'Unknown\'), \' (Code: \', if(greater(length(activity(\'Pre-Ingest Cleanup\').output.errors), 0), activity(\'Pre-Ingest Cleanup\').output.errors[0].Code, \'None\'), \')\')'
- type: 'Expression'
- }
- errorCode: 'DataExplorerPreIngestionDropFailed'
- }
- }
- { // Abort On Post-Ingest Drop Error
- name: 'Abort On Post-Ingest Drop Error'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Post-Ingest Cleanup'
- dependencyConditions: [
- 'Failed'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'tryAgain'
- value: false
- }
- }
- { // Error: DataExplorerPostIngestionDropFailed
- name: 'Post-Ingest Drop Failed Error'
- type: 'Fail'
- dependsOn: [
- {
- activity: 'Abort On Post-Ingest Drop Error'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- message: {
- value: '@concat(\'Data Explorer post-ingestion cleanup (drop extents from final tables) for the \', replace(pipeline().parameters.table, \'_raw\', \'_final_*\'), \' table failed. Please fix the error and rerun ingestion for the following folder path: "\', pipeline().parameters.folderPath, \'". File: \', pipeline().parameters.originalFileName, \'. Error: \', if(greater(length(activity(\'Post-Ingest Cleanup\').output.errors), 0), activity(\'Post-Ingest Cleanup\').output.errors[0].Message, \'Unknown\'), \' (Code: \', if(greater(length(activity(\'Post-Ingest Cleanup\').output.errors), 0), activity(\'Post-Ingest Cleanup\').output.errors[0].Code, \'None\'), \')\')'
- type: 'Expression'
- }
- errorCode: 'DataExplorerPostIngestionDropFailed'
- }
- }
- ]
- }
- }
- ]
- timeout: '0.02:00:00'
- }
- }
- ]
- parameters: {
- folderPath: {
- type: 'string'
- }
- fileName: {
- type: 'string'
- }
- originalFileName: {
- type: 'string'
- }
- ingestionId: {
- type: 'string'
- }
- table: {
- type: 'string'
- }
- }
- variables: {
- tryAgain: {
- type: 'Boolean'
- defaultValue: true
- }
- logRetentionDays: {
- type: 'Integer'
- defaultValue: 0
- }
- finalRetentionMonths: {
- type: 'Integer'
- defaultValue: 999
- }
- }
- annotations: []
- }
-}
-
-//------------------------------------------------------------------------------
-// ingestion_ExecuteETL pipeline
-// Triggered by ingestion_ManifestAdded trigger
-//------------------------------------------------------------------------------
-@description('Queues the ingestion_ETL_dataExplorer pipeline to account for Data Factory pipeline trigger limits.')
-resource pipeline_ExecuteIngestionETL 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = if (deployDataExplorer || useFabric) {
- name: '${safeIngestionContainerName}_ExecuteETL'
- parent: dataFactory
- properties: {
- concurrency: 1
- activities: [
- { // Wait
- name: 'Wait'
- description: 'Files may not be available immediately after being created.'
- type: 'Wait'
- dependsOn: []
- userProperties: []
- typeProperties: {
- waitTimeInSeconds: 60
- }
- }
- { // Set Container Folder Path
- name: 'Set Container Folder Path'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Wait'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'containerFolderPath'
- value: {
- value: '@join(skip(array(split(pipeline().parameters.folderPath, \'/\')), 1), \'/\')'
- type: 'Expression'
- }
- }
- }
- { // Get Existing Parquet Files
- name: 'Get Existing Parquet Files'
- description: 'Get the previously ingested files so we can get file paths.'
- type: 'GetMetadata'
- dependsOn: [
- {
- activity: 'Set Container Folder Path'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- timeout: '0.12:00:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- dataset: {
- referenceName: dataset_ingestion_files.name
- type: 'DatasetReference'
- parameters: {
- folderPath: '@variables(\'containerFolderPath\')'
- }
- }
- fieldList: [
- 'childItems'
- ]
- storeSettings: {
- type: 'AzureBlobFSReadSettings'
- enablePartitionDiscovery: false
- }
- formatSettings: {
- type: 'ParquetReadSettings'
- }
- }
- }
- { // Filter Out Folders and manifest files
- name: 'Filter Out Folders'
- description: 'Remove any folders or manifest files.'
- type: 'Filter'
- dependsOn: [
- {
- activity: 'Get Existing Parquet Files'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- items: {
- value: '@if(contains(activity(\'Get Existing Parquet Files\').output, \'childItems\'), activity(\'Get Existing Parquet Files\').output.childItems, json(\'[]\'))'
- type: 'Expression'
- }
- condition: {
- value: '@and(equals(item().type, \'File\'), not(contains(toLower(item().name), \'manifest.json\')))'
- type: 'Expression'
- }
- }
- }
- { // Set Ingestion Timestamp
- name: 'Set Ingestion Timestamp'
- type: 'SetVariable'
- dependsOn: [
- {
- activity: 'Wait'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- variableName: 'timestamp'
- value: {
- value: '@utcNow()'
- type: 'Expression'
- }
- }
- }
- { // For Each Old File
- name: 'For Each Old File'
- description: 'Loop thru each of the existing files.'
- type: 'ForEach'
- dependsOn: [
- {
- activity: 'Filter Out Folders'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- {
- activity: 'Data Explorer validation'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- batchCount: dataExplorerIngestionCapacity // Concurrency limit
- items: {
- value: '@activity(\'Filter Out Folders\').output.Value'
- type: 'Expression'
- }
- activities: [
- { // Execute
- name: 'Execute'
- description: 'Run the ADX ETL pipeline.'
- type: 'ExecutePipeline'
- dependsOn: []
- policy: {
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- pipeline: {
- referenceName: pipeline_ToDataExplorer.name
- type: 'PipelineReference'
- }
- waitOnCompletion: true
- parameters: {
- folderPath: {
- value: '@variables(\'containerFolderPath\')'
- type: 'Expression'
- }
- fileName: {
- value: '@item().name'
- type: 'Expression'
- }
- originalFileName: {
- value: '@last(array(split(item().name, \'${ingestionIdFileNameSeparator}\')))'
- type: 'Expression'
- }
- ingestionId: {
- value: '@concat(first(array(split(item().name, \'${ingestionIdFileNameSeparator}\'))), \'_\', variables(\'timestamp\'))'
- type: 'Expression'
- }
- table: {
- value: '@concat(first(array(split(variables(\'containerFolderPath\'), \'/\'))), \'_raw\')'
- type: 'Expression'
- }
- }
- }
- }
- ]
- }
- }
- { // If No Files
- name: 'If No Files'
- description: 'If there are no files found, fail the pipeline.'
- type: 'IfCondition'
- dependsOn: [
- {
- activity: 'Filter Out Folders'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- expression: {
- value: '@equals(length(activity(\'Filter Out Folders\').output.Value), 0)'
- type: 'Expression'
- }
- ifTrueActivities: [
- { // Error: IngestionFilesNotFound
- name: 'Files Not Found'
- type: 'Fail'
- dependsOn: []
- userProperties: []
- typeProperties: {
- message: {
- value: '@concat(\'Unable to locate parquet files to ingest from the \', pipeline().parameters.folderPath, \' path. Please confirm the folder path is the full path, including the "ingestion" container and not starting with or ending with a slash ("/").\')'
- type: 'Expression'
- }
- errorCode: 'IngestionFilesNotFound'
- }
- }
- ]
- }
- }
- {
- name: 'Data Explorer validation'
- description: 'If Data Explorer is stopped, start it'
- type: 'IfCondition'
- dependsOn: [
- {
- activity: 'Set Ingestion Timestamp'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- expression: {
- value: '@equals(${deployDataExplorer}, true)'
- type: 'Expression'
- }
- ifTrueActivities: [
- {
- name: 'Start ADX Cluster'
- type: 'WebActivity'
- dependsOn: []
- policy: {
- timeout: '0.12:00:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- method: 'POST'
- url: {
- value: '${environment().resourceManager}${dataExplorerCluster.id}/start?api-version=2024-04-13'
- type: 'Expression'
- }
- body: '{}'
- authentication: {
- type: 'MSI'
- resource: {
- value: environment().resourceManager
- type: 'Expression'
- }
- }
- }
- }
- {
- name: 'Error ADX Start'
- type: 'Fail'
- dependsOn: [
- {
- activity: 'Start ADX Cluster After Error'
- dependencyConditions: [
- 'Failed'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- message: {
- value:'@concat(\'Failed to start the Data Explorer instance. Message: \', activity(\'Start ADX Cluster After Error\').output.error.message)'
- type: 'Expression'
- }
- errorCode: {
- value: '@activity(\'Start ADX Cluster After Error\').output.error.code'
- type: 'Expression'
- }
- }
- }
- {
- name: 'Wait ADX Provision State'
- type: 'Wait'
- dependsOn: [
- {
- activity: 'Start ADX Cluster'
- dependencyConditions: [
- 'Failed'
- ]
- }
- ]
- userProperties: []
- typeProperties: {
- waitTimeInSeconds: 600
- }
- }
- {
- name: 'Start ADX Cluster After Error'
- type: 'WebActivity'
- dependsOn: [
- {
- activity: 'Wait ADX Provision State'
- dependencyConditions: [
- 'Succeeded'
- ]
- }
- ]
- policy: {
- timeout: '0.12:00:00'
- retry: 0
- retryIntervalInSeconds: 30
- secureOutput: false
- secureInput: false
- }
- userProperties: []
- typeProperties: {
- method: 'POST'
- url: {
- value: '${environment().resourceManager}${dataExplorerCluster.id}/start?api-version=2024-04-13'
- type: 'Expression'
- body: '{}'
- }
- authentication: {
- type: 'MSI'
- resource: {
- value: environment().resourceManager
- type: 'Expression'
- }
- }
- }
- }
- ]
- }
- }
- ]
- parameters: {
- folderPath: {
- type: 'string'
- }
- }
- variables: {
- containerFolderPath: {
- type: 'string'
- }
- timestamp: {
- type: 'string'
- }
- }
- annotations: [
- 'New ingestion'
- ]
- }
-}
-
-//------------------------------------------------------------------------------
-// Start all triggers
-//------------------------------------------------------------------------------
-
-module startTriggers 'hub-deploymentScript.bicep' = {
- name: 'Microsoft.FinOpsHubs.Core_ADF.StartTriggers'
- dependsOn: [
- triggerManagerRoleAssignments
- trigger_ExportManifestAdded
- trigger_IngestionManifestAdded
- trigger_SettingsUpdated
- trigger_DailySchedule
- trigger_MonthlySchedule
- deleteOldResources
- ]
- params: {
- app: app
- identityName: triggerManagerIdentity.name
- scriptContent: loadTextContent('./scripts/Start-Triggers.ps1')
- environmentVariables: [
- {
- name: 'DataFactorySubscriptionId'
- value: subscription().id
- }
- {
- name: 'DataFactoryResourceGroup'
- value: resourceGroup().name
- }
- {
- name: 'DataFactoryName'
- value: dataFactory.name
- }
- {
- name: 'Triggers'
- value: join(allHubTriggers, '|')
- }
- {
- name: 'Pipelines'
- value: join([ pipeline_InitializeHub.name ], '|')
- }
- ]
- }
-}
-
-//==============================================================================
-// Outputs
-//==============================================================================
-
-@description('The Resource ID of the Data factory.')
-output resourceId string = dataFactory.id
-
-@description('The Name of the Azure Data Factory instance.')
-output name string = dataFactory.name
diff --git a/src/templates/finops-hub/modules/ftkver.txt b/src/templates/finops-hub/modules/fx/ftkver.txt
similarity index 100%
rename from src/templates/finops-hub/modules/ftkver.txt
rename to src/templates/finops-hub/modules/fx/ftkver.txt
diff --git a/src/templates/finops-hub/modules/fx/hub-app.bicep b/src/templates/finops-hub/modules/fx/hub-app.bicep
new file mode 100644
index 000000000..4eefc1a5e
--- /dev/null
+++ b/src/templates/finops-hub/modules/fx/hub-app.bicep
@@ -0,0 +1,576 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { getAppPublisherTags, HubAppProperties, HubAppFeature } from 'hub-types.bicep'
+
+
+//==============================================================================
+// Parameters
+//==============================================================================
+
+@description('Required. FinOps hub app getting deployed.')
+param app HubAppProperties
+
+@description('Required. Version number of the FinOps hub app.')
+param version string
+
+// @description('Required. Minimum version number supported by the FinOps hub app.')
+// param hubMinVersion string
+
+// @description('Required. Maximum version number supported by the FinOps hub app.')
+// param hubMaxVersion string
+
+@description('Optional. Indicate which features the app requires. Allowed values: "DataFactory", "KeyVault", "Storage". Default: [] (none).')
+param features HubAppFeature[] = []
+
+@description('Optional. Indicate which RBAC roles the Data Factory identity needs on the storage account, if created. This is in addition to Storage Blob Data Contributor for reading and managing content. Default: [] (none).')
+param storageRoles string[] = []
+
+@description('Optional. Custom string with additional metadata to log. Must an alphanumeric string without spaces or special characters except for underscores and dashes. Namespace + appName + telemetryString must be 50 characters or less - additional characters will be trimmed.')
+param telemetryString string = ''
+
+
+//==============================================================================
+// Variables
+//==============================================================================
+
+// Features
+var usesDataFactory = contains(features, 'DataFactory')
+var usesKeyVault = contains(features, 'KeyVault')
+var usesStorage = contains(features, 'Storage')
+
+// App telemetry
+var telemetryId = 'ftk-hubapp-${app.id}${empty(telemetryString) ? '' : '_'}${telemetryString}' // cSpell:ignore hubapp
+var telemetryProps = {
+ mode: 'Incremental'
+ template: {
+ '$schema': 'https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#'
+ contentVersion: '1.0.0.0'
+ metadata: {
+ _generator: {
+ name: 'FTK: ${app.id}'
+ version: version
+ }
+ }
+ resources: []
+ }
+}
+
+// Roles needed to auto-start Data Factory triggers
+var autoStartRbacRoles = [
+ // Data Factory contributor -- https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#data-factory-contributor
+ // Used to start/stop triggers and delete old pipelines/triggers
+ '673868aa-7521-48a0-acc6-0f60742d39f5'
+]
+
+// Roles for ADF to manage data in storage
+// Does not include roles assignments needed against the export scope
+var factoryStorageRoles = union(storageRoles, [
+ // Storage Account Contributor -- https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#storage-account-contributor
+ // Used to move files from the msexports to ingestion container
+ '17d1049b-9a84-46fb-8f53-869881c3d3ab'
+ // Storage Blob Data Contributor -- https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#storage-blob-data-contributor
+ 'ba92f5b4-2d11-453d-a403-e96b0029c9fe'
+ // Reader -- https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#reader
+ 'acdd72a7-3385-48ef-bd42-f606fba81ae7'
+])
+
+// Storage infrastructure encryption
+var storageInfrastructureEncryptionProperties = !app.hub.options.storageInfrastructureEncryption ? {} : {
+ encryption: {
+ keySource: 'Microsoft.Storage'
+ requireInfrastructureEncryption: app.hub.options.storageInfrastructureEncryption
+ }
+}
+
+// KeyVault access policies
+var keyVaultAccessPolicies = [
+ {
+ #disable-next-line BCP318 // Null safety warning for conditional resource access // Null safety warning for conditional resource access
+ objectId: dataFactory.identity.principalId
+ tenantId: subscription().tenantId
+ permissions: { secrets: ['get'] }
+ }
+]
+
+
+//==============================================================================
+// Resources
+//==============================================================================
+
+// TODO: Get hub instance to verify version compatibility
+
+//------------------------------------------------------------------------------
+// Telemetry
+// Used to anonymously count the number of times the template has been deployed
+// and to track and fix deployment bugs to ensure the highest quality.
+// No information about you or your cost data is collected.
+//------------------------------------------------------------------------------
+
+resource appTelemetry 'Microsoft.Resources/deployments@2022-09-01' = if (app.hub.options.enableTelemetry) {
+ name: length(telemetryId) <= 64 ? telemetryId : substring(telemetryId, 0, 64)
+ tags: getAppPublisherTags(app, 'Microsoft.Resources/deployments')
+ properties: telemetryProps
+}
+
+//------------------------------------------------------------------------------
+// Data Factory
+//------------------------------------------------------------------------------
+
+resource dataFactory 'Microsoft.DataFactory/factories@2018-06-01' = if (usesDataFactory) {
+ name: app.dataFactory
+ location: app.hub.location
+ tags: getAppPublisherTags(app, 'Microsoft.DataFactory/factories')
+ identity: { type: 'SystemAssigned' }
+ properties: any({ // Using any() to hide the error that gets surfaced because globalConfigurations is not in the ADF schema yet
+ globalConfigurations: {
+ PipelineBillingEnabled: 'true'
+ }
+ })
+
+ resource managedVirtualNetwork 'managedVirtualNetworks' = if (app.hub.options.privateRouting) {
+ name: 'default'
+ properties: {}
+
+ resource storageManagedPrivateEndpoint 'managedPrivateEndpoints' = if (usesStorage) {
+ name: storageAccount.name
+ properties: {
+ #disable-next-line BCP318 // Null safety warning for conditional resource access // Null safety warning for conditional resource access // Null safety warning for conditional resource access
+ name: storageAccount.name
+ groupId: 'dfs'
+ #disable-next-line BCP318 // Null safety warning for conditional resource access // Null safety warning for conditional resource access // Null safety warning for conditional resource access
+ privateLinkResourceId: storageAccount.id
+ fqdns: [
+ #disable-next-line BCP318 // Null safety warning for conditional resource access // Null safety warning for conditional resource access // Null safety warning for conditional resource access // Null safety warning for conditional resource access
+ storageAccount.properties.primaryEndpoints.dfs
+ ]
+ }
+ }
+
+ resource keyVaultManagedPrivateEndpoint 'managedPrivateEndpoints' = if (usesKeyVault) {
+ #disable-next-line BCP318 // Null safety warning for conditional resource access // Null safety warning for conditional resource access
+ name: keyVault.name
+ properties: {
+ #disable-next-line BCP318 // Null safety warning for conditional resource access // Null safety warning for conditional resource access // Null safety warning for conditional resource access
+ name: keyVault.name
+ groupId: 'vault'
+ #disable-next-line BCP318 // Null safety warning for conditional resource access // Null safety warning for conditional resource access // Null safety warning for conditional resource access
+ privateLinkResourceId: keyVault.id
+ fqdns: [
+ #disable-next-line BCP318 // Null safety warning for conditional resource access // Null safety warning for conditional resource access // Null safety warning for conditional resource access // Null safety warning for conditional resource access
+ keyVault.properties.vaultUri
+ ]
+ }
+ }
+ }
+
+ resource managedIntegrationRuntime 'integrationRuntimes' = if (app.hub.options.privateRouting) {
+ name: 'ManagedIntegrationRuntime'
+
+ properties: {
+ type: 'Managed'
+ managedVirtualNetwork: {
+ referenceName: dataFactory::managedVirtualNetwork.name
+ type: 'ManagedVirtualNetworkReference'
+ }
+ typeProperties: {
+ computeProperties: {
+ location: app.hub.location
+ dataFlowProperties: {
+ computeType: 'General'
+ coreCount: 8
+ timeToLive: 10
+ cleanup: false
+ customProperties: []
+ }
+ copyComputeScaleProperties: {
+ dataIntegrationUnit: 16
+ timeToLive: 30
+ }
+ pipelineExternalComputeScaleProperties: {
+ timeToLive: 30
+ numberOfPipelineNodes: 1
+ numberOfExternalNodes: 1
+ }
+ }
+ }
+ }
+ }
+
+ // cSpell:ignore linkedservices
+ resource linkedService_keyVault 'linkedservices' = if (usesKeyVault) {
+ name: keyVault.name
+ dependsOn: app.hub.options.privateRouting ? [managedIntegrationRuntime] : []
+ properties: {
+ annotations: []
+ parameters: {}
+ type: 'AzureKeyVault'
+ typeProperties: {
+ baseUrl: reference('Microsoft.KeyVault/vaults/${keyVault.name}', '2023-02-01').vaultUri
+ }
+ connectVia: app.hub.options.privateRouting
+ ? {
+ referenceName: managedIntegrationRuntime.name
+ type: 'IntegrationRuntimeReference'
+ }
+ : null
+ }
+ }
+
+ resource linkedService_storageAccount 'linkedservices' = if (usesStorage) {
+ name: storageAccount.name
+ dependsOn: app.hub.options.privateRouting ? [managedIntegrationRuntime] : []
+ properties: {
+ annotations: []
+ parameters: {}
+ type: 'AzureBlobFS'
+ typeProperties: {
+ url: reference('Microsoft.Storage/storageAccounts/${storageAccount.name}', '2021-08-01').primaryEndpoints.dfs
+ }
+ connectVia: app.hub.options.privateRouting
+ ? {
+ referenceName: managedIntegrationRuntime.name
+ type: 'IntegrationRuntimeReference'
+ }
+ : null
+ }
+ }
+}
+
+// TODO: Consolidate keyVaultEndpoints.bicep into hub-app.bicep
+module getKeyVaultPrivateEndpointConnections 'keyVaultEndpoints.bicep' = if (usesDataFactory && usesKeyVault && app.hub.options.privateRouting) {
+ name: 'GetKeyVaultPrivateEndpointConnections'
+ dependsOn: [
+ dataFactory::managedVirtualNetwork::keyVaultManagedPrivateEndpoint
+ getStoragePrivateEndpointConnections // Queue Key Vault private endpoints after storage since we can only run one deployment at a time with private endpoints
+ ]
+ params: {
+ keyVaultName: keyVault.name
+ }
+}
+
+module approveKeyVaultPrivateEndpointConnections 'keyVaultEndpoints.bicep' = if (usesDataFactory && usesKeyVault && app.hub.options.privateRouting) {
+ name: 'ApproveKeyVaultPrivateEndpointConnections'
+ params: {
+ keyVaultName: keyVault.name
+ #disable-next-line BCP318 // Null safety warning for conditional resource access // Null safety warning for conditional resource access
+ privateEndpointConnections: getKeyVaultPrivateEndpointConnections.outputs.privateEndpointConnections
+ }
+}
+
+// TODO: Consolidate storageEndpoints.bicep into hub-app.bicep
+module getStoragePrivateEndpointConnections 'storageEndpoints.bicep' = if (usesDataFactory && usesStorage && app.hub.options.privateRouting) {
+ name: 'GetStoragePrivateEndpointConnections'
+ dependsOn: [
+ dataFactory::managedVirtualNetwork::storageManagedPrivateEndpoint
+ stopTriggers // Queue storage private endpoints after triggers are stopped since we can only run one deployment at a time with private endpoints
+ ]
+ params: {
+ storageAccountName: storageAccount.name
+ }
+}
+
+module approveStoragePrivateEndpointConnections 'storageEndpoints.bicep' = if (usesDataFactory && usesStorage && app.hub.options.privateRouting) {
+ name: 'ApproveStoragePrivateEndpointConnections'
+ params: {
+ storageAccountName: storageAccount.name
+ #disable-next-line BCP318 // Null safety warning for conditional resource access // Null safety warning for conditional resource access
+ privateEndpointConnections: getStoragePrivateEndpointConnections.outputs.privateEndpointConnections
+ }
+}
+
+//------------------------------------------------------------------------------
+// Role assignments
+//------------------------------------------------------------------------------
+
+// Grant ADF identity access to storage
+resource storageRoleAssignments 'Microsoft.Authorization/roleAssignments@2022-04-01' = [
+ for role in factoryStorageRoles: {
+ name: guid(storageAccount.id, role, dataFactory.id)
+ scope: storageAccount
+ properties: {
+ roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', role)
+ #disable-next-line BCP318 // Null safety warning for conditional resource access // Null safety warning for conditional resource access
+ principalId: dataFactory.identity.principalId
+ principalType: 'ServicePrincipal'
+ }
+ }
+]
+
+//------------------------------------------------------------------------------
+// Stop triggers and delete old resources
+//------------------------------------------------------------------------------
+
+// Create managed identity to start/stop triggers
+resource triggerManagerIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = if (usesDataFactory) {
+ name: '${dataFactory.name}_triggerManager'
+ location: app.hub.location
+ tags: union(app.tags, app.hub.tagsByResource[?'Microsoft.ManagedIdentity/userAssignedIdentities'] ?? {})
+}
+
+resource triggerManagerRoleAssignments 'Microsoft.Authorization/roleAssignments@2022-04-01' = [
+ for role in autoStartRbacRoles: if (usesDataFactory) {
+ name: guid(dataFactory.id, role, triggerManagerIdentity.id)
+ scope: dataFactory
+ properties: {
+ roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', role)
+ #disable-next-line BCP318 // Null safety warning for conditional resource access // Null safety warning for conditional resource access
+ principalId: triggerManagerIdentity.properties.principalId
+ principalType: 'ServicePrincipal'
+ }
+ }
+]
+
+// Stop all triggers before deploying triggers
+module stopTriggers 'hub-deploymentScript.bicep' = {
+ name: '${app.publisher}.${app.name}_ADF.StopTriggers'
+ dependsOn: [
+ // TODO: Do we need to make this optional only if private endpoints are enabled and telemetry is enabled? Will it fail when telemetry is disabled?
+ appTelemetry // Ensure the telemetry deployment is run before stopping triggers since we can only run one deployment at a time with private endpoints
+ triggerManagerRoleAssignments
+ ]
+ params: {
+ app: app
+ identityName: triggerManagerIdentity.name
+ scriptContent: loadTextContent('./scripts/Init-DataFactory.ps1')
+ arguments: '-Stop'
+ environmentVariables: [
+ {
+ name: 'DataFactorySubscriptionId'
+ value: subscription().id
+ }
+ {
+ name: 'DataFactoryResourceGroup'
+ value: resourceGroup().name
+ }
+ {
+ name: 'DataFactoryName'
+ #disable-next-line BCP318 // Null safety warning for conditional resource access // Null safety warning for conditional resource access // Null safety warning for conditional resource access
+ value: dataFactory.name
+ }
+ ]
+ }
+}
+
+//------------------------------------------------------------------------------
+// Storage account
+//------------------------------------------------------------------------------
+
+resource storageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' = if (usesStorage) {
+ name: app.storage
+ location: app.hub.location
+ sku: {
+ name: app.hub.options.storageSku
+ }
+ kind: 'BlockBlobStorage'
+ tags: getAppPublisherTags(app, 'Microsoft.Storage/storageAccounts')
+ properties: {
+ ...storageInfrastructureEncryptionProperties
+ supportsHttpsTrafficOnly: true
+ allowSharedKeyAccess: true
+ isHnsEnabled: true
+ minimumTlsVersion: 'TLS1_2'
+ allowBlobPublicAccess: false
+ publicNetworkAccess: 'Enabled'
+ networkAcls: {
+ bypass: 'AzureServices'
+ defaultAction: app.hub.options.privateRouting ? 'Deny' : 'Allow'
+ }
+ }
+
+ resource blobService 'blobServices' = {
+ name: 'default'
+ }
+}
+
+resource blobPrivateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' existing = if (usesStorage && app.hub.options.privateRouting) {
+ name: 'privatelink.blob.${environment().suffixes.storage}' // cSpell:ignore privatelink
+}
+
+resource blobEndpoint 'Microsoft.Network/privateEndpoints@2023-11-01' = if (usesStorage && app.hub.options.privateRouting) {
+ name: '${storageAccount.name}-blob-ep'
+ location: app.hub.location
+ tags: getAppPublisherTags(app, 'Microsoft.Network/privateEndpoints')
+ properties: {
+ subnet: {
+ id: app.hub.routing.subnets.storage
+ }
+ privateLinkServiceConnections: [
+ {
+ name: 'blobLink'
+ properties: {
+ #disable-next-line BCP318 // Null safety warning for conditional resource access // Null safety warning for conditional resource access // Null safety warning for conditional resource access // Null safety warning for conditional resource access
+ privateLinkServiceId: storageAccount.id
+ groupIds: ['blob']
+ }
+ }
+ ]
+ }
+
+ resource blobPrivateDnsZoneGroup 'privateDnsZoneGroups' = {
+ name: 'storage-endpoint-zone'
+ properties: {
+ privateDnsZoneConfigs: [
+ {
+ name: blobPrivateDnsZone.name
+ properties: {
+ privateDnsZoneId: blobPrivateDnsZone.id
+ }
+ }
+ ]
+ }
+ }
+}
+
+resource dfsPrivateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' existing = if (usesStorage && app.hub.options.privateRouting) {
+ name: 'privatelink.dfs.${environment().suffixes.storage}' // cSpell:ignore privatelink
+}
+
+resource dfsEndpoint 'Microsoft.Network/privateEndpoints@2023-11-01' = if (usesStorage && app.hub.options.privateRouting) {
+ name: '${storageAccount.name}-dfs-ep'
+ location: app.hub.location
+ tags: getAppPublisherTags(app, 'Microsoft.Network/privateEndpoints')
+ properties: {
+ subnet: {
+ id: app.hub.routing.subnets.storage
+ }
+ privateLinkServiceConnections: [
+ {
+ name: 'dfsLink'
+ properties: {
+ #disable-next-line BCP318 // Null safety warning for conditional resource access // Null safety warning for conditional resource access // Null safety warning for conditional resource access // Null safety warning for conditional resource access
+ privateLinkServiceId: storageAccount.id
+ groupIds: ['dfs']
+ }
+ }
+ ]
+ }
+
+ resource dfsPrivateDnsZoneGroup 'privateDnsZoneGroups' = {
+ name: 'dfs-endpoint-zone'
+ properties: {
+ privateDnsZoneConfigs: [
+ {
+ name: dfsPrivateDnsZone.name
+ properties: {
+ privateDnsZoneId: dfsPrivateDnsZone.id
+ }
+ }
+ ]
+ }
+ }
+}
+
+//------------------------------------------------------------------------------
+// KeyVault for secrets
+//------------------------------------------------------------------------------
+
+resource keyVault 'Microsoft.KeyVault/vaults@2023-02-01' = if (usesKeyVault) {
+ name: app.keyVault
+ location: app.hub.location
+ tags: getAppPublisherTags(app, 'Microsoft.KeyVault/vaults')
+ properties: {
+ sku: any({
+ name: app.hub.options.keyVaultSku
+ family: 'A'
+ })
+ enabledForDeployment: true
+ enabledForTemplateDeployment: true
+ enabledForDiskEncryption: true
+ enableSoftDelete: true
+ softDeleteRetentionInDays: 90
+ enableRbacAuthorization: false
+ createMode: 'default'
+ tenantId: subscription().tenantId
+ accessPolicies: keyVaultAccessPolicies
+ networkAcls: {
+ bypass: 'AzureServices'
+ defaultAction: app.hub.options.privateRouting ? 'Deny' : 'Allow'
+ }
+ }
+
+ resource keyVault_accessPolicies 'accessPolicies' = {
+ name: 'add'
+ properties: {
+ accessPolicies: keyVaultAccessPolicies
+ }
+ }
+}
+
+resource keyVaultPrivateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' = if (usesKeyVault && app.hub.options.privateRouting) {
+ name: 'privatelink${replace(environment().suffixes.keyvaultDns, 'vault', 'vaultcore')}' // cSpell:ignore privatelink, vaultcore
+ location: 'global'
+ tags: getAppPublisherTags(app, 'Microsoft.Network/privateDnsZones')
+ properties: {}
+
+ resource keyVaultPrivateDnsZoneLink 'virtualNetworkLinks@2024-06-01' = {
+ name: '${replace(keyVaultPrivateDnsZone.name, '.', '-')}-link'
+ location: 'global'
+ tags: getAppPublisherTags(app, 'Microsoft.Network/privateDnsZones/virtualNetworkLinks')
+ properties: {
+ virtualNetwork: {
+ id: app.hub.routing.networkId
+ }
+ registrationEnabled: false
+ }
+ }
+}
+
+resource keyVaultEndpoint 'Microsoft.Network/privateEndpoints@2023-11-01' = if (usesKeyVault && app.hub.options.privateRouting) {
+ name: '${keyVault.name}-ep'
+ location: app.hub.location
+ tags: getAppPublisherTags(app, 'Microsoft.Network/privateEndpoints')
+ properties: {
+ subnet: {
+ id: app.hub.routing.subnets.keyVault
+ }
+ privateLinkServiceConnections: [
+ {
+ name: 'keyVaultLink'
+ properties: {
+ #disable-next-line BCP318 // Null safety warning for conditional resource access // Null safety warning for conditional resource access // Null safety warning for conditional resource access // Null safety warning for conditional resource access
+ privateLinkServiceId: keyVault.id
+ groupIds: ['vault']
+ }
+ }
+ ]
+ }
+
+ resource keyVaultPrivateDnsZoneGroup 'privateDnsZoneGroups' = {
+ name: 'keyvault-endpoint-zone'
+ properties: {
+ privateDnsZoneConfigs: [
+ {
+ name: keyVaultPrivateDnsZone.name
+ properties: {
+ privateDnsZoneId: keyVaultPrivateDnsZone.id
+ }
+ }
+ ]
+ }
+ }
+}
+
+
+//==============================================================================
+// Outputs
+//==============================================================================
+
+@description('Resource ID of the Data Factory instance used by the FinOps hub app.')
+#disable-next-line BCP318 // Null safety warning for conditional resource access
+output dataFactoryId string = dataFactory.id
+
+@description('Resource ID of the Key Vault instance used by the FinOps hub app.')
+#disable-next-line BCP318 // Null safety warning for conditional resource access
+output keyVaultId string = keyVault.id
+
+@description('Resource ID of the storage account instance used by the FinOps hub app.')
+#disable-next-line BCP318 // Null safety warning for conditional resource access
+output storageAccountId string = storageAccount.id
+
+@description('Principal ID for the managed identity used by Data Factory.')
+#disable-next-line BCP318 // Null safety warning for conditional resource access
+output principalId string = dataFactory.identity.principalId
+
+@description('Name of the managed identity used to create and stop ADF triggers.')
+output triggerManagerIdentityName string = triggerManagerIdentity.name
diff --git a/src/templates/finops-hub/modules/hub-database.bicep b/src/templates/finops-hub/modules/fx/hub-database.bicep
similarity index 93%
rename from src/templates/finops-hub/modules/hub-database.bicep
rename to src/templates/finops-hub/modules/fx/hub-database.bicep
index 085e53b0f..871b6a966 100644
--- a/src/templates/finops-hub/modules/hub-database.bicep
+++ b/src/templates/finops-hub/modules/fx/hub-database.bicep
@@ -34,6 +34,7 @@ resource cluster 'Microsoft.Kusto/clusters@2023-08-15' existing = {
resource script 'scripts' = [for scr in items(scripts) : {
name: scr.key
properties: {
+ #disable-next-line use-secure-value-for-secure-inputs // KQL scripts don't contain sensitive information
scriptContent: scr.value
continueOnErrors: continueOnErrors
forceUpdateTag: forceUpdateTag
diff --git a/src/templates/finops-hub/modules/hub-deploymentScript.bicep b/src/templates/finops-hub/modules/fx/hub-deploymentScript.bicep
similarity index 95%
rename from src/templates/finops-hub/modules/hub-deploymentScript.bicep
rename to src/templates/finops-hub/modules/fx/hub-deploymentScript.bicep
index 6fdc804fc..95408c527 100644
--- a/src/templates/finops-hub/modules/hub-deploymentScript.bicep
+++ b/src/templates/finops-hub/modules/fx/hub-deploymentScript.bicep
@@ -40,12 +40,13 @@ param environmentVariables EnvironmentVariable[] = []
var privateEndpointDeploymentRoles = !app.hub.options.privateRouting ? [] : [
'69566ab7-960f-475b-8e7c-b3118f30c6bd' // Storage File Data Privileged Contributor - https://learn.microsoft.com/azure/role-based-access-control/built-in-roles/storage#storage-file-data-privileged-contributor
]
+var containerGroupName = replace(replace(replace(scriptName, '/', '-'), '.', '-'), '_', '-')
var privateEndpointDeploymentProperties = !app.hub.options.privateRouting ? {} : {
storageAccountSettings: {
storageAccountName: app.hub.routing.scriptStorage ?? ''
}
containerSettings: {
- containerGroupName: '${app.hub.routing.scriptStorage}cg'
+ containerGroupName: length(containerGroupName) > 63 ? substring(containerGroupName, 0, 62) : containerGroupName
subnetIds: [
{
id: app.hub.routing.subnets.scripts ?? ''
@@ -110,7 +111,7 @@ resource script 'Microsoft.Resources/deploymentScripts@2023-08-01' = {
}
properties: {
...privateEndpointDeploymentProperties
- azPowerShellVersion: '9.0'
+ azPowerShellVersion: '11.0'
retentionInterval: 'PT1H'
cleanupPreference: 'OnSuccess'
scriptContent: scriptContent
diff --git a/src/templates/finops-hub/modules/hub-event-trigger.bicep b/src/templates/finops-hub/modules/fx/hub-eventTrigger.bicep
similarity index 100%
rename from src/templates/finops-hub/modules/hub-event-trigger.bicep
rename to src/templates/finops-hub/modules/fx/hub-eventTrigger.bicep
diff --git a/src/templates/finops-hub/modules/hub-identity.bicep b/src/templates/finops-hub/modules/fx/hub-identity.bicep
similarity index 92%
rename from src/templates/finops-hub/modules/hub-identity.bicep
rename to src/templates/finops-hub/modules/fx/hub-identity.bicep
index f440e0e14..0b12f2776 100644
--- a/src/templates/finops-hub/modules/hub-identity.bicep
+++ b/src/templates/finops-hub/modules/fx/hub-identity.bicep
@@ -1,7 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
-import { getPublisherTags, HubAppProperties } from 'hub-types.bicep'
+import { getAppPublisherTags, HubAppProperties } from 'hub-types.bicep'
//==============================================================================
@@ -28,7 +28,7 @@ param roles string[]
// Create managed identity
resource identity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = {
name: identityName
- tags: getPublisherTags(app, 'Microsoft.ManagedIdentity/userAssignedIdentities')
+ tags: getAppPublisherTags(app, 'Microsoft.ManagedIdentity/userAssignedIdentities')
location: app.hub.location
}
diff --git a/src/templates/finops-hub/modules/fx/hub-initialize.bicep b/src/templates/finops-hub/modules/fx/hub-initialize.bicep
new file mode 100644
index 000000000..b688f8a59
--- /dev/null
+++ b/src/templates/finops-hub/modules/fx/hub-initialize.bicep
@@ -0,0 +1,76 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { HubAppProperties } from 'hub-types.bicep'
+
+
+//==============================================================================
+// Parameters
+//==============================================================================
+
+@description('Required. FinOps hub app getting deployed.')
+param app HubAppProperties
+
+@description('Required. List of Azure Data Factory instances to start triggers for. Can be up to 1 per publisher.')
+param dataFactoryInstances string[]
+
+@description('Required. Name of the managed identity to use when starting the triggers.')
+param identityName string
+
+@description('Optional. Start all triggers for the Data Factory instances. Default: false.')
+param startAllTriggers bool = false
+
+@description('Optional. List of pipelines to run. Default: [] (no pipelines).')
+param startPipelines string[] = []
+
+
+//==============================================================================
+// Variables
+//==============================================================================
+
+// Clean up dataFactoryInstances array - remove empty values and duplicates
+var uniqueInstances = union(filter(dataFactoryInstances, adf => !empty(adf)), [])
+
+//==============================================================================
+// Resources
+//==============================================================================
+
+// Initialize Data Factory instances (start triggers and/or run pipelines)
+module initialize 'hub-deploymentScript.bicep' = [
+ for adf in uniqueInstances: {
+ name: length('Microsoft.FinOpsHubs.Init_${adf}') <= 64 ? 'Microsoft.FinOpsHubs.Init_${adf}' : substring('Microsoft.FinOpsHubs.Init_${adf}', 0, 64)
+ params: {
+ app: app
+ identityName: identityName
+ scriptContent: loadTextContent('./scripts/Init-DataFactory.ps1')
+ environmentVariables: [
+ {
+ name: 'DataFactorySubscriptionId'
+ value: subscription().id
+ }
+ {
+ name: 'DataFactoryResourceGroup'
+ value: resourceGroup().name
+ }
+ {
+ name: 'DataFactoryName'
+ value: adf
+ }
+ {
+ name: 'Pipelines'
+ value: join(startPipelines, '|')
+ }
+ {
+ name: 'StartAllTriggers'
+ value: string(startAllTriggers)
+ }
+ ]
+ }
+ }
+]
+
+//==============================================================================
+// Outputs
+//==============================================================================
+
+// None
diff --git a/src/templates/finops-hub/modules/hub-storage.bicep b/src/templates/finops-hub/modules/fx/hub-storage.bicep
similarity index 93%
rename from src/templates/finops-hub/modules/hub-storage.bicep
rename to src/templates/finops-hub/modules/fx/hub-storage.bicep
index 723628f44..8dda03fe0 100644
--- a/src/templates/finops-hub/modules/hub-storage.bicep
+++ b/src/templates/finops-hub/modules/fx/hub-storage.bicep
@@ -3,7 +3,6 @@
import { HubAppProperties } from 'hub-types.bicep'
-
//==============================================================================
// Parameters
//==============================================================================
@@ -20,7 +19,6 @@ param files object = {}
@description('Optional. Indicates whether to create the blob manager user assigned identity even if files are not being uploaded. Default: false.')
param forceCreateBlobManagerIdentity bool = false
-
//==============================================================================
// Variables
//==============================================================================
@@ -28,7 +26,6 @@ param forceCreateBlobManagerIdentity bool = false
var fileCount = length(items(files))
var hasFiles = fileCount > 0
-
//==============================================================================
// Resources
//==============================================================================
@@ -36,7 +33,7 @@ var hasFiles = fileCount > 0
// Get storage account instance
resource storageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' existing = {
name: app.storage
-
+
resource blobService 'blobServices@2022-09-01' existing = {
name: 'default'
@@ -63,7 +60,7 @@ module identity 'hub-identity.bicep' = if (hasFiles || forceCreateBlobManagerIde
// Storage Blob Data Contributor - https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#storage-blob-data-contributor
// Used by deployment scripts to write data to blob storage
'ba92f5b4-2d11-453d-a403-e96b0029c9fe'
-
+
// Storage File Data Privileged Contributor - https://learn.microsoft.com/azure/role-based-access-control/built-in-roles/storage#storage-file-data-privileged-contributor
// https://learn.microsoft.com/azure/azure-resource-manager/templates/deployment-script-template#use-existing-storage-account
'69566ab7-960f-475b-8e7c-b3118f30c6bd'
@@ -76,6 +73,7 @@ module uploadFiles 'hub-deploymentScript.bicep' = if (hasFiles) {
name: '${deployment().name}.Upload'
params: {
app: app
+ #disable-next-line BCP318 // Null safety warning for conditional resource access
identityName: identity.outputs.name
environmentVariables: [
{
@@ -95,7 +93,6 @@ module uploadFiles 'hub-deploymentScript.bicep' = if (hasFiles) {
}
}
-
//==============================================================================
// Outputs
//==============================================================================
@@ -107,10 +104,13 @@ output containerName string = storageAccount::blobService::targetContainer.name
output filesUploaded int = fileCount
@description('Resource ID of the user assigned identity used to upload files. Will be empty if no files are uploaded or forceCreateBlobManagerIdentity is false.')
+#disable-next-line BCP318 // Null safety warning for conditional resource access
output identityId string = hasFiles || forceCreateBlobManagerIdentity ? identity.outputs.id : ''
@description('Name of the user assigned identity used to upload files. Will be empty if no files are uploaded or forceCreateBlobManagerIdentity is false.')
+#disable-next-line BCP318 // Null safety warning for conditional resource access
output identityName string = hasFiles || forceCreateBlobManagerIdentity ? identity.outputs.name : ''
@description('Principal ID of the user assigned identity used to upload files. Will be empty if no files are uploaded or forceCreateBlobManagerIdentity is false.')
+#disable-next-line BCP318 // Null safety warning for conditional resource access
output identityPrincipalId string = hasFiles || forceCreateBlobManagerIdentity ? identity.outputs.principalId : ''
diff --git a/src/templates/finops-hub/modules/hub-types.bicep b/src/templates/finops-hub/modules/fx/hub-types.bicep
similarity index 77%
rename from src/templates/finops-hub/modules/hub-types.bicep
rename to src/templates/finops-hub/modules/fx/hub-types.bicep
index b4b2fc771..d0072df2c 100644
--- a/src/templates/finops-hub/modules/hub-types.bicep
+++ b/src/templates/finops-hub/modules/fx/hub-types.bicep
@@ -32,6 +32,7 @@ type IdNameObject = { id: string, name: string }
table: 'Resource ID and name for the table storage DNS zone.'
}
subnets: {
+ dataExplorer: 'Resource ID of the subnet for the Data Explorer instance.'
dataFactory: 'Resource ID of the subnet for Data Factory instances.'
keyVault: 'Resource ID of the subnet for Key Vault instances.'
scripts: 'Resource ID of the subnet for deployment script storage.'
@@ -49,6 +50,7 @@ type HubRoutingProperties = {
table: IdNameObject
}
subnets: {
+ dataExplorer: string
dataFactory: string
keyVault: string
scripts: string
@@ -110,34 +112,26 @@ type HubProperties = {
@export()
@description('FinOps hub app configuration settings.')
@metadata({
- name: 'Short name of the FinOps hub app (not including the publisher namespace).'
- displayName: 'Display name of the FinOps hub app.'
- tags: 'Tags to apply to all FinOps hub resources for this FinOps hub app.'
- publisher: {
- name: 'Fully-qualified namespace of the FinOps hub app publisher.'
- displayName: 'Display name of the FinOps hub app publisher.'
- suffix: 'Unique suffix used for publisher resources.'
- tags: 'Tags to apply to all FinOps hub resources for this FinOps hub app publisher.'
- }
- hub: 'FinOps hub instance the app is deployed to.'
+ id: 'Fully-qualified name of the publisher and app, separated by a dot.'
+ name: 'Short name of the FinOps hub app. Last segment of the app ID.'
+ publisher: 'Fully-qualified namespace of the FinOps hub app publisher.'
+ suffix: 'Unique suffix used for publisher resources.'
+ tags: 'Tags to apply to all FinOps hub resources for this FinOps hub app publisher. Tags are not specific to the app since resources are shared.'
dataFactory: 'Name of the Data Factory instance for this publisher.'
keyVault: 'Name of the KeyVault instance for this publisher.'
storage: 'Name of the storage account for this publisher.'
+ hub: 'FinOps hub instance the app is deployed to.'
})
type HubAppProperties = {
+ id: string
name: string
- displayName: string
+ publisher: string
+ suffix: string
tags: object
- publisher: {
- name: string
- displayName: string
- suffix: string
- tags: object
- }
- hub: HubProperties
dataFactory: string
keyVault: string
storage: string
+ hub: HubProperties
}
@export()
@@ -149,6 +143,7 @@ type HubAppFeature = 'DataFactory' | 'KeyVault' | 'Storage'
// Variables
//==============================================================================
+@export()
@description('Version of the FinOps toolkit.')
var finOpsToolkitVersion = loadTextContent('ftkver.txt') // cSpell:ignore ftkver
@@ -221,10 +216,11 @@ func newHubInternal(
table: enablePublicAccess ? { id:'', name:'' } : dnsZoneIdName('table')
}
subnets: {
- dataFactory: enablePublicAccess ? '' : resourceId('Microsoft.Network/virtualNetworks/subnets', networkName, 'private-endpoint-subnet')!
- keyVault: enablePublicAccess ? '' : resourceId('Microsoft.Network/virtualNetworks/subnets', networkName, 'private-endpoint-subnet')!
- scripts: enablePublicAccess ? '' : resourceId('Microsoft.Network/virtualNetworks/subnets', networkName, 'script-subnet')!
- storage: enablePublicAccess ? '' : resourceId('Microsoft.Network/virtualNetworks/subnets', networkName, 'private-endpoint-subnet')!
+ dataExplorer: enablePublicAccess ? '' : resourceId('Microsoft.Network/virtualNetworks/subnets', networkName, 'dataExplorer-subnet')!
+ dataFactory: enablePublicAccess ? '' : resourceId('Microsoft.Network/virtualNetworks/subnets', networkName, 'private-endpoint-subnet')!
+ keyVault: enablePublicAccess ? '' : resourceId('Microsoft.Network/virtualNetworks/subnets', networkName, 'private-endpoint-subnet')!
+ scripts: enablePublicAccess ? '' : resourceId('Microsoft.Network/virtualNetworks/subnets', networkName, 'script-subnet')!
+ storage: enablePublicAccess ? '' : resourceId('Microsoft.Network/virtualNetworks/subnets', networkName, 'private-endpoint-subnet')!
}
}
core: {
@@ -268,56 +264,48 @@ func newHub(
// Internal function to create a new FinOps hub configuration object that includes extra parameters that are reused within the function.
func newAppInternal(
hub HubProperties,
- publisherName string,
- publisherDisplayName string,
- publisherSuffix string,
- publisherTags object,
- appName string,
- appDisplayName string,
- version string,
+ id string,
+ name string,
+ publisher string,
+ suffix string,
) HubAppProperties => {
- name: appName
- displayName: appDisplayName
- tags: union(hub.tags, publisherTags, {
- 'ftk-hubapp': appName // cSpell:ignore hubapp
- 'ftk-hubapp-version': version
- })
- publisher: {
- name: publisherName
- displayName: publisherDisplayName
- suffix: publisherSuffix
- tags: union(hub.tags, publisherTags)
- }
+ id: id
+ name: name
+ publisher: publisher
+ suffix: suffix
+ tags: union(
+ hub.tags,
+ { 'ftk-hubapp-publisher': publisher } // publisherTags
+ // TODO: How do we want to handle app-specific tags?
+ // {
+ // 'ftk-hubapp': appName // cSpell:ignore hubapp
+ // 'ftk-hubapp-version': version
+ // }
+ )
hub: hub
// Globally unique Data Factory name: 3-63 chars; letters, numbers, non-repeating dashes
- dataFactory: replace('${take('${replace(hub.name, '_', '-')}-engine', 63 - length(publisherSuffix) - 1)}-${publisherSuffix}', '--', '-')
+ dataFactory: replace('${take('${replace(hub.name, '_', '-')}-engine', 63 - length(suffix) - 1)}-${suffix}', '--', '-')
// Globally unique KeyVault name: 3-24 chars; letters, numbers, dashes
- keyVault: replace('${take('${replace(hub.name, '_', '-')}-vault', 24 - length(publisherSuffix) - 1)}-${publisherSuffix}', '--', '-')
+ keyVault: replace('${take('${replace(hub.name, '_', '-')}-vault', 24 - length(suffix) - 1)}-${suffix}', '--', '-')
// Globally unique storage account name: 3-24 chars; lowercase letters/numbers only
- storage: '${take(safeStorageName(hub.name), 24 - length(publisherSuffix))}${publisherSuffix}'
+ storage: '${take(safeStorageName(hub.name), 24 - length(suffix))}${suffix}'
}
@export()
@description('Creates a new FinOps hub app configuration object.')
func newApp(
hub HubProperties,
- publisherDisplayName string,
- publisherName string,
- appPartialName string,
- appDisplayName string,
- version string,
+ publisher string,
+ app string,
) HubAppProperties => newAppInternal(
hub,
- publisherName,
- publisherDisplayName,
- !hub.options.publisherIsolation || publisherName == 'Microsoft.FinOpsHubs' ? hub.core.suffix : uniqueString(publisherName), // publisherSuffix
- { 'ftk-hubapp-publisher': publisherName }, // publisherTags
- '${publisherName}.${appPartialName}', // appName
- appDisplayName,
- version
+ '${publisher}.${app}', // id
+ app,
+ publisher,
+ !hub.options.publisherIsolation || publisher == 'Microsoft.FinOpsHubs' ? hub.core.suffix : uniqueString(publisher) // publisher suffix
)
@export()
@@ -329,14 +317,21 @@ func getHubTags(hub HubProperties, resourceType string) object => union(
@export()
@description('Returns a tags dictionary that includes tags for the FinOps hub app publisher.')
-func getPublisherTags(app HubAppProperties, resourceType string) object => union(
- app.hub.options.publisherIsolation ? app.publisher.tags : app.hub.tags,
+func getAppPublisherTags(app HubAppProperties, resourceType string) object => union(
+ app.hub.options.publisherIsolation ? app.tags : app.hub.tags,
app.hub.tagsByResource[?resourceType] ?? {}
)
+
+//------------------------------------------------------------------------------
+// Private routing
+//------------------------------------------------------------------------------
+
@export()
-@description('Returns a tags dictionary that includes tags for the FinOps hub app.')
-func getAppTags(app HubAppProperties, resourceType string, forceAppTags bool?) object => union(
- app.hub.options.publisherIsolation || (forceAppTags ?? false) ? app.tags : app.hub.tags,
- app.hub.tagsByResource[?resourceType] ?? {}
-)
+@description('Returns an object that represents the properties needed to enable private routing for linked services. Use property expansion (`...value`) to apply to a linkedServices resource.')
+func privateRoutingForLinkedServices(hub HubProperties) object => hub.options.privateRouting ? {
+ connectVia: {
+ referenceName: 'ManagedIntegrationRuntime'
+ type: 'IntegrationRuntimeReference'
+ }
+} : {}
diff --git a/src/templates/finops-hub/modules/hub-vault.bicep b/src/templates/finops-hub/modules/fx/hub-vault.bicep
similarity index 100%
rename from src/templates/finops-hub/modules/hub-vault.bicep
rename to src/templates/finops-hub/modules/fx/hub-vault.bicep
diff --git a/src/templates/finops-hub/modules/keyVaultEndpoints.bicep b/src/templates/finops-hub/modules/fx/keyVaultEndpoints.bicep
similarity index 100%
rename from src/templates/finops-hub/modules/keyVaultEndpoints.bicep
rename to src/templates/finops-hub/modules/fx/keyVaultEndpoints.bicep
diff --git a/src/templates/finops-hub/modules/fx/scripts/Init-DataFactory.ps1 b/src/templates/finops-hub/modules/fx/scripts/Init-DataFactory.ps1
new file mode 100644
index 000000000..6ec94421c
--- /dev/null
+++ b/src/templates/finops-hub/modules/fx/scripts/Init-DataFactory.ps1
@@ -0,0 +1,73 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+param(
+ [switch] $Stop
+)
+
+# Init outputs
+$DeploymentScriptOutputs = @{}
+
+if (-not $Stop)
+{
+ Start-Sleep -Seconds 10
+}
+
+# Loop thru triggers
+$triggers = Get-AzDataFactoryV2Trigger `
+ -ResourceGroupName $env:DataFactoryResourceGroup `
+ -DataFactoryName $env:DataFactoryName
+
+Write-Output "Found $($triggers.Length) trigger(s)"
+
+if ($startTriggers)
+{
+ $triggers | ForEach-Object {
+ $trigger = $_.Name
+ if ($Stop)
+ {
+ Write-Output "Stopping trigger $trigger..."
+ $triggerOutput = Stop-AzDataFactoryV2Trigger `
+ -ResourceGroupName $env:DataFactoryResourceGroup `
+ -DataFactoryName $env:DataFactoryName `
+ -Name $trigger `
+ -Force `
+ -ErrorAction SilentlyContinue # Ignore errors, since the trigger may not exist
+ }
+ else
+ {
+ Write-Output "Starting trigger $trigger..."
+ $triggerOutput = Start-AzDataFactoryV2Trigger `
+ -ResourceGroupName $env:DataFactoryResourceGroup `
+ -DataFactoryName $env:DataFactoryName `
+ -Name $trigger `
+ -Force
+ }
+ if ($triggerOutput)
+ {
+ Write-Output "done..."
+ }
+ else
+ {
+ Write-Output "failed..."
+ }
+ $DeploymentScriptOutputs[$trigger] = $triggerOutput
+ }
+
+ if ($Stop)
+ {
+ Start-Sleep -Seconds 10
+ }
+}
+
+if (-not [string]::IsNullOrWhiteSpace($env:Pipelines))
+{
+ $env:Pipelines.Split('|') `
+ | ForEach-Object {
+ Write-Output "Running the init pipeline..."
+ Invoke-AzDataFactoryV2Pipeline `
+ -ResourceGroupName $env:DataFactoryResourceGroup `
+ -DataFactoryName $env:DataFactoryName `
+ -PipelineName $_
+ }
+}
diff --git a/src/templates/finops-hub/modules/scripts/Remove-OldResources.ps1 b/src/templates/finops-hub/modules/fx/scripts/Remove-OldResources.ps1
similarity index 100%
rename from src/templates/finops-hub/modules/scripts/Remove-OldResources.ps1
rename to src/templates/finops-hub/modules/fx/scripts/Remove-OldResources.ps1
diff --git a/src/templates/finops-hub/modules/scripts/Upload-StorageFile.ps1 b/src/templates/finops-hub/modules/fx/scripts/Upload-StorageFile.ps1
similarity index 100%
rename from src/templates/finops-hub/modules/scripts/Upload-StorageFile.ps1
rename to src/templates/finops-hub/modules/fx/scripts/Upload-StorageFile.ps1
diff --git a/src/templates/finops-hub/modules/storageEndpoints.bicep b/src/templates/finops-hub/modules/fx/storageEndpoints.bicep
similarity index 100%
rename from src/templates/finops-hub/modules/storageEndpoints.bicep
rename to src/templates/finops-hub/modules/fx/storageEndpoints.bicep
diff --git a/src/templates/finops-hub/modules/hub-app.bicep b/src/templates/finops-hub/modules/hub-app.bicep
deleted file mode 100644
index 932e12688..000000000
--- a/src/templates/finops-hub/modules/hub-app.bicep
+++ /dev/null
@@ -1,332 +0,0 @@
-// Copyright (c) Microsoft Corporation.
-// Licensed under the MIT License.
-
-import { getAppTags, getPublisherTags, HubAppProperties, HubAppFeature, HubProperties, newApp } from 'hub-types.bicep'
-
-
-//==============================================================================
-// Parameters
-//==============================================================================
-
-@description('Required. FinOps hub instance properties.')
-param hub HubProperties
-
-@description('Required. Display name of the FinOps hub app publisher.')
-param publisher string
-
-@description('Required. Namespace to use for the FinOps hub app publisher. Will be combined with appName to form a fully-qualified identifier. Must be an alphanumeric string without spaces or special characters except for periods. This value should never change and will be used to uniquely identify the publisher. A change would require migrating content to the new publisher. Namespace + appName + telemetryString must be 50 characters or less - additional characters will be trimmed.')
-param namespace string
-
-@description('Required. Unique identifier of the FinOps hub app within the publisher namespace. Must be an alphanumeric string without spaces or special characters. This name should never change and will be used with the namespace to fully qualify the app. A change would require migrating content to the new app. Namespace + appName + telemetryString must be 50 characters or less - additional characters will be trimmed.')
-param appName string
-
-@description('Required. Display name of the FinOps hub app.')
-param displayName string
-
-@description('Optional. Version number of the FinOps hub app.')
-param appVersion string = ''
-
-// @description('Required. Minimum version number supported by the FinOps hub app.')
-// param hubMinVersion string
-
-// @description('Required. Maximum version number supported by the FinOps hub app.')
-// param hubMaxVersion string
-
-@description('Optional. Indicate which features the app requires. Allowed values: "Storage". Default: [] (none).')
-param features HubAppFeature[] = []
-
-@description('Optional. Custom string with additional metadata to log. Must an alphanumeric string without spaces or special characters except for underscores and dashes. Namespace + appName + telemetryString must be 50 characters or less - additional characters will be trimmed.')
-param telemetryString string = ''
-
-
-//==============================================================================
-// Variables
-//==============================================================================
-
-var app = newApp(hub, publisher, namespace, appName, displayName, appVersion)
-
-// Features
-var usesDataFactory = contains(features, 'DataFactory')
-var usesKeyVault = contains(features, 'KeyVault')
-var usesStorage = contains(features, 'Storage')
-
-// App telemetry
-var telemetryId = 'ftk-hubapp-${app.name}${empty(telemetryString) ? '' : '_'}${telemetryString}' // cSpell:ignore hubapp
-var telemetryProps = {
- mode: 'Incremental'
- template: {
- '$schema': 'https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#'
- contentVersion: '1.0.0.0'
- metadata: {
- _generator: {
- name: 'FTK: ${publisher} - ${displayName} ${telemetryId}'
- version: appVersion
- }
- }
- resources: []
- }
-}
-
-// Storage infrastructure encryption
-var storageInfrastructureEncryptionProperties = !hub.options.storageInfrastructureEncryption ? {} : {
- encryption: {
- keySource: 'Microsoft.Storage'
- requireInfrastructureEncryption: hub.options.storageInfrastructureEncryption
- }
-}
-
-// KeyVault access policies
-var keyVaultAccessPolicies = [
- {
- objectId: dataFactory.identity.principalId
- tenantId: subscription().tenantId
- permissions: { secrets: ['get'] }
- }
-]
-
-
-//==============================================================================
-// Resources
-//==============================================================================
-
-// TODO: Get hub instance to verify version compatibility
-
-//------------------------------------------------------------------------------
-// Telemetry
-// Used to anonymously count the number of times the template has been deployed
-// and to track and fix deployment bugs to ensure the highest quality.
-// No information about you or your cost data is collected.
-//------------------------------------------------------------------------------
-
-resource appTelemetry 'Microsoft.Resources/deployments@2022-09-01' = if (hub.options.enableTelemetry) {
- name: length(telemetryId) <= 64 ? telemetryId : substring(telemetryId, 0, 64)
- tags: getAppTags(app, 'Microsoft.Resources/deployments', true)
- properties: telemetryProps
-}
-
-//------------------------------------------------------------------------------
-// TODO: Get hub details
-//------------------------------------------------------------------------------
-
-//------------------------------------------------------------------------------
-// Data Factory
-//------------------------------------------------------------------------------
-
-resource dataFactory 'Microsoft.DataFactory/factories@2018-06-01' = if (usesDataFactory) {
- name: app.dataFactory
- location: app.hub.location
- tags: getPublisherTags(app, 'Microsoft.DataFactory/factories')
- identity: { type: 'SystemAssigned' }
- properties: any({ // Using any() to hide the error that gets surfaced because globalConfigurations is not in the ADF schema yet
- globalConfigurations: {
- PipelineBillingEnabled: 'true'
- }
- })
-}
-
-//------------------------------------------------------------------------------
-// Storage account
-//------------------------------------------------------------------------------
-
-resource storageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' = if (usesStorage) {
- name: app.storage
- location: hub.location
- sku: {
- name: hub.options.storageSku
- }
- kind: 'BlockBlobStorage'
- tags: getPublisherTags(app, 'Microsoft.Storage/storageAccounts')
- properties: {
- ...storageInfrastructureEncryptionProperties
- supportsHttpsTrafficOnly: true
- allowSharedKeyAccess: true
- isHnsEnabled: true
- minimumTlsVersion: 'TLS1_2'
- allowBlobPublicAccess: false
- publicNetworkAccess: 'Enabled'
- networkAcls: {
- bypass: 'AzureServices'
- defaultAction: hub.options.privateRouting ? 'Deny' : 'Allow'
- }
- }
-
- resource blobService 'blobServices' = {
- name: 'default'
- }
-}
-
-resource blobPrivateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' existing = if (usesStorage && hub.options.privateRouting) {
- name: 'privatelink.blob.${environment().suffixes.storage}' // cSpell:ignore privatelink
-}
-
-resource blobEndpoint 'Microsoft.Network/privateEndpoints@2023-11-01' = if (usesStorage && hub.options.privateRouting) {
- name: '${storageAccount.name}-blob-ep'
- location: hub.location
- tags: getPublisherTags(app, 'Microsoft.Network/privateEndpoints')
- properties: {
- subnet: {
- id: hub.routing.subnets.storage
- }
- privateLinkServiceConnections: [
- {
- name: 'blobLink'
- properties: {
- privateLinkServiceId: storageAccount.id
- groupIds: ['blob']
- }
- }
- ]
- }
-
- resource blobPrivateDnsZoneGroup 'privateDnsZoneGroups' = {
- name: 'storage-endpoint-zone'
- properties: {
- privateDnsZoneConfigs: [
- {
- name: blobPrivateDnsZone.name
- properties: {
- privateDnsZoneId: blobPrivateDnsZone.id
- }
- }
- ]
- }
- }
-}
-
-resource dfsPrivateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' existing = if (usesStorage && hub.options.privateRouting) {
- name: 'privatelink.dfs.${environment().suffixes.storage}' // cSpell:ignore privatelink
-}
-
-resource dfsEndpoint 'Microsoft.Network/privateEndpoints@2023-11-01' = if (usesStorage && hub.options.privateRouting) {
- name: '${storageAccount.name}-dfs-ep'
- location: hub.location
- tags: getPublisherTags(app, 'Microsoft.Network/privateEndpoints')
- properties: {
- subnet: {
- id: hub.routing.subnets.storage
- }
- privateLinkServiceConnections: [
- {
- name: 'dfsLink'
- properties: {
- privateLinkServiceId: storageAccount.id
- groupIds: ['dfs']
- }
- }
- ]
- }
-
- resource dfsPrivateDnsZoneGroup 'privateDnsZoneGroups' = {
- name: 'dfs-endpoint-zone'
- properties: {
- privateDnsZoneConfigs: [
- {
- name: dfsPrivateDnsZone.name
- properties: {
- privateDnsZoneId: dfsPrivateDnsZone.id
- }
- }
- ]
- }
- }
-}
-
-//------------------------------------------------------------------------------
-// KeyVault for secrets
-//------------------------------------------------------------------------------
-
-resource keyVault 'Microsoft.KeyVault/vaults@2023-02-01' = if (usesKeyVault) {
- name: app.keyVault
- location: hub.location
- tags: getPublisherTags(app, 'Microsoft.KeyVault/vaults')
- properties: {
- sku: any({
- name: hub.options.keyVaultSku
- family: 'A'
- })
- enabledForDeployment: true
- enabledForTemplateDeployment: true
- enabledForDiskEncryption: true
- enableSoftDelete: true
- softDeleteRetentionInDays: 90
- enableRbacAuthorization: false
- createMode: 'default'
- tenantId: subscription().tenantId
- accessPolicies: keyVaultAccessPolicies
- networkAcls: {
- bypass: 'AzureServices'
- defaultAction: hub.options.privateRouting ? 'Deny' : 'Allow'
- }
- }
-
- resource keyVault_accessPolicies 'accessPolicies' = {
- name: 'add'
- properties: {
- accessPolicies: keyVaultAccessPolicies
- }
- }
-}
-
-resource keyVaultPrivateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' = if (usesKeyVault && hub.options.privateRouting) {
- name: 'privatelink${replace(environment().suffixes.keyvaultDns, 'vault', 'vaultcore')}' // cSpell:ignore privatelink, vaultcore
- location: 'global'
- tags: getPublisherTags(app, 'Microsoft.Network/privateDnsZones')
- properties: {}
-
- resource keyVaultPrivateDnsZoneLink 'virtualNetworkLinks@2024-06-01' = {
- name: '${replace(keyVaultPrivateDnsZone.name, '.', '-')}-link'
- location: 'global'
- tags: getPublisherTags(app, 'Microsoft.Network/privateDnsZones/virtualNetworkLinks')
- properties: {
- virtualNetwork: {
- id: hub.routing.networkId
- }
- registrationEnabled: false
- }
- }
-}
-
-resource keyVaultEndpoint 'Microsoft.Network/privateEndpoints@2023-11-01' = if (usesKeyVault && hub.options.privateRouting) {
- name: '${keyVault.name}-ep'
- location: hub.location
- tags: getPublisherTags(app, 'Microsoft.Network/privateEndpoints')
- properties: {
- subnet: {
- id: hub.routing.subnets.keyVault
- }
- privateLinkServiceConnections: [
- {
- name: 'keyVaultLink'
- properties: {
- privateLinkServiceId: keyVault.id
- groupIds: ['vault']
- }
- }
- ]
- }
-
- resource keyVaultPrivateDnsZoneGroup 'privateDnsZoneGroups' = {
- name: 'keyvault-endpoint-zone'
- properties: {
- privateDnsZoneConfigs: [
- {
- name: keyVaultPrivateDnsZone.name
- properties: {
- privateDnsZoneId: keyVaultPrivateDnsZone.id
- }
- }
- ]
- }
- }
-}
-
-
-//==============================================================================
-// Outputs
-//==============================================================================
-
-@description('FinOps hub app configuration.')
-output app HubAppProperties = app
-
-@description('Principal ID for the managed identity used by Data Factory.')
-output principalId string = dataFactory.identity.principalId
diff --git a/src/templates/finops-hub/modules/hub.bicep b/src/templates/finops-hub/modules/hub.bicep
index 5929bda87..5c5793106 100644
--- a/src/templates/finops-hub/modules/hub.bicep
+++ b/src/templates/finops-hub/modules/hub.bicep
@@ -1,7 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
-import { getHubTags, newApp, newHub } from 'hub-types.bicep'
+import { getHubTags, newApp, newHub } from 'fx/hub-types.bicep'
//==============================================================================
@@ -40,7 +40,7 @@ param remoteHubStorageUri string = ''
@description('Optional. Storage account key for remote storage account.')
@secure()
param remoteHubStorageKey string = ''
-
+
@description('Optional. Enable managed exports where your FinOps hub instance will create and run Cost Management exports on your behalf. Not supported for Microsoft Customer Agreement (MCA) billing profiles. Requires the ability to grant User Access Administrator role to FinOps hubs, which is required to create Cost Management exports. Default: true.')
param enableManagedExports bool = true
@@ -72,7 +72,7 @@ param dataExplorerName string = ''
'Standard_DS13_v2+2TB_PS'
'Standard_DS14_v2+3TB_PS'
'Standard_DS14_v2+4TB_PS'
- 'Standard_E2a_v4' // 2 CPU, 14GB RAM, 78GB cache, $220/mo
+ 'Standard_E2a_v4' // 2 CPU, 14GB RAM, 78GB cache, $220/mo
'Standard_E2ads_v5'
'Standard_E2d_v4'
'Standard_E2d_v5'
@@ -185,45 +185,8 @@ var hub = newHub(
enableDefaultTelemetry
)
-// Do not reference these deployments directly or indirectly to avoid a DeploymentNotFound error
var useFabric = !empty(fabricQueryUri)
-var deployDataExplorer = !useFabric && !empty(dataExplorerName)
-var safeDataExplorerName = !deployDataExplorer ? '' : dataExplorer.outputs.clusterName
-var safeDataExplorerUri = useFabric ? fabricQueryUri : (!deployDataExplorer ? '' : dataExplorer.outputs.clusterUri)
-var safeDataExplorerId = !deployDataExplorer ? '' : dataExplorer.outputs.clusterId
-var safeDataExplorerIngestionDb = useFabric ? 'Ingestion' : (!deployDataExplorer ? '' : dataExplorer.outputs.ingestionDbName)
-var safeDataExplorerIngestionCapacity = useFabric ? fabricCapacityUnits : (!deployDataExplorer ? 1 : dataExplorer.outputs.clusterIngestionCapacity)
-var safeDataExplorerPrincipalId = !deployDataExplorer ? '' : dataExplorer.outputs.principalId
-var safeVnetId = enablePublicAccess ? '' : infrastructure.outputs.vNetId
-var safeDataExplorerSubnetId = enablePublicAccess ? '' : infrastructure.outputs.dataExplorerSubnetId
-// var safeFinopsHubSubnetId = enablePublicAccess ? '' : infrastructure.outputs.finopsHubSubnetId
-// var safeScriptSubnetId = enablePublicAccess ? '' : infrastructure.outputs.scriptSubnetId
-
-// cSpell:ignore eventgrid
-// var eventGridName = 'finops-hub-eventgrid-${config.hub.suffix}'
-
-// var eventGridPrefix = '${replace(hubName, '_', '-')}-ns'
-// var eventGridSuffix = '-${config.hub.suffix}'
-// var eventGridName = replace(
-// '${take(eventGridPrefix, 50 - length(eventGridSuffix))}${eventGridSuffix}',
-// '--',
-// '-'
-// )
-
-// EventGrid Contributor role
-// var eventGridContributorRoleId = '1e241071-0855-49ea-94dc-649edcd759de'
-
-// cSpell:ignore israelcentral, uaenorth, italynorth, switzerlandnorth, mexicocentral, southcentralus, polandcentral, swedencentral, spaincentral, francecentral, usdodeast, usdodcentral
-// Find a fallback region for EventGrid
-// var eventGridLocationFallback = {
-// israelcentral: 'uaenorth'
-// italynorth: 'switzerlandnorth'
-// mexicocentral: 'southcentralus'
-// polandcentral: 'swedencentral'
-// spaincentral: 'francecentral'
-// usdodeast: 'usdodcentral'
-// }
-// var finalEventGridLocation = eventGridLocation != null && !empty(eventGridLocation) ? eventGridLocation : (eventGridLocationFallback[?location] ?? location)
+var useAzureDataExplorer = !useFabric && !empty(dataExplorerName) // Prefer Fabric over Azure Data Explorer
// The last segment of the GUID in the telemetryId (40b) is used to identify this module
// Remaining characters identify settings; must be <= 12 chars -- Example: (guid)_RLXD##x1000P
@@ -236,11 +199,11 @@ var telemetryString = join([
// F = Fabric enabled
!useFabric ? '' : 'F${fabricCapacityUnits}'
// X = ADX enabled + D (dev) or S (standard) SKU
- !deployDataExplorer ? '' : 'X${substring(dataExplorerSku, 0, 1)}'
+ !useAzureDataExplorer ? '' : 'X${substring(dataExplorerSku, 0, 1)}'
// Number of cores in the VM size
- !deployDataExplorer ? '' : replace(replace(replace(replace(replace(replace(replace(replace(split(split(dataExplorerSku, 'Standard_')[1], '_')[0], 'C', ''), 'D', ''), 'E', ''), 'L', ''), 'a', ''), 'd', ''), 'i', ''), 's', '')
+ !useAzureDataExplorer ? '' : replace(replace(replace(replace(replace(replace(replace(replace(split(split(dataExplorerSku, 'Standard_')[1], '_')[0], 'C', ''), 'D', ''), 'E', ''), 'L', ''), 'a', ''), 'd', ''), 'i', ''), 's', '')
// Number of nodes in the cluster
- !deployDataExplorer || dataExplorerCapacity == 1 ? '' : 'x${dataExplorerCapacity}'
+ !useAzureDataExplorer || dataExplorerCapacity == 1 ? '' : 'x${dataExplorerCapacity}'
// P = private endpoints enabled
enablePublicAccess ? '' : 'P'
], '')
@@ -265,7 +228,7 @@ resource telemetry 'Microsoft.Resources/deployments@2022-09-01' = if (enableDefa
metadata: {
_generator: {
name: 'FinOps toolkit'
- version: loadTextContent('ftkver.txt') // cSpell:ignore ftkver
+ version: loadTextContent('fx/ftkver.txt') // cSpell:ignore ftkver
}
}
resources: []
@@ -273,30 +236,14 @@ resource telemetry 'Microsoft.Resources/deployments@2022-09-01' = if (enableDefa
}
}
-//------------------------------------------------------------------------------
-// Base resources needed for hub apps
-//------------------------------------------------------------------------------
-
-// TODO: Can this be merged into core.bicep?
-module infrastructure 'infrastructure.bicep' = {
- name: 'Microsoft.FinOpsHubs.Infrastructure'
- params: {
- hub: hub
- }
-}
-
//------------------------------------------------------------------------------
// Hub core app
//------------------------------------------------------------------------------
-module core 'core.bicep' = {
+module core 'Microsoft.FinOpsHubs/Core/app.bicep' = {
name: 'Microsoft.FinOpsHubs.Core'
- dependsOn: [
- infrastructure
- ]
params: {
- hub: hub
- telemetryString: telemetryString
+ app: newApp(hub, 'Microsoft.FinOpsHubs', 'Core')
scopesToMonitor: scopesToMonitor
msexportRetentionInDays: exportRetentionInDays // cSpell:ignore msexport
ingestionRetentionInMonths: ingestionRetentionInMonths
@@ -306,16 +253,26 @@ module core 'core.bicep' = {
}
//------------------------------------------------------------------------------
-// ADLSv2 storage account for staging and archive
+// Cost Management
//------------------------------------------------------------------------------
-module cmExports 'cm-exports.bicep' = {
+module cmExports 'Microsoft.CostManagement/Exports/app.bicep' = {
name: 'Microsoft.CostManagement.Exports'
dependsOn: [
core
]
params: {
- hub: hub
+ app: newApp(hub, 'Microsoft.CostManagement', 'Exports')
+ }
+}
+
+module cmManagedExports 'Microsoft.CostManagement/ManagedExports/app.bicep' = if (enableManagedExports) {
+ name: 'Microsoft.CostManagement.ManagedExports'
+ dependsOn: [
+ cmExports
+ ]
+ params: {
+ app: newApp(hub, 'Microsoft.CostManagement', 'ManagedExports')
}
}
@@ -323,75 +280,83 @@ module cmExports 'cm-exports.bicep' = {
// Data Explorer for analytics
//------------------------------------------------------------------------------
-module dataExplorer 'dataExplorer.bicep' = if (deployDataExplorer) {
- name: 'dataExplorer'
+module analytics 'Microsoft.FinOpsHubs/Analytics/app.bicep' = if (useFabric || useAzureDataExplorer) {
+ name: 'Microsoft.FinOpsHubs.Analytics'
+ dependsOn: hub.options.privateRouting ? [
+ core
+ // When private endpoints are enabled, we need to explicitly block on anything that uses deployment scripts to guarantee only one deployment script runs at a time
+ cmExports
+ deleteOldResources
+ ] : [
+ core
+ ]
params: {
+ app: newApp(hub, 'Microsoft.FinOpsHubs', 'Analytics')
+ fabricQueryUri: fabricQueryUri
+ fabricCapacityUnits: fabricCapacityUnits
clusterName: dataExplorerName
clusterSku: dataExplorerSku
clusterCapacity: dataExplorerCapacity
- // TODO: Figure out why this is breaking upgrades -- clusterTrustedExternalTenants: dataExplorerTrustedExternalTenants
- location: location
- tags: hub.tags
- tagsByResource: tagsByResource
- dataFactoryName: core.outputs.dataFactoryName
rawRetentionInDays: dataExplorerRawRetentionInDays
- virtualNetworkId: safeVnetId // cSpell:ignore vnet
- privateEndpointSubnetId: safeDataExplorerSubnetId
- enablePublicAccess: enablePublicAccess
- storageAccountName: core.outputs.storageAccountName
+ // TODO: Figure out why this is breaking upgrades -- clusterTrustedExternalTenants: dataExplorerTrustedExternalTenants
}
}
//------------------------------------------------------------------------------
-// Data Factory and pipelines
+// Remote hub app
//------------------------------------------------------------------------------
-module dataFactoryResources 'dataFactory.bicep' = {
- name: 'dataFactoryResources'
+module remoteHub 'Microsoft.FinOpsHubs/RemoteHub/app.bicep' = if (!empty(remoteHubStorageKey)) {
+ name: 'Microsoft.FinOpsHubs.RemoteHub'
+ dependsOn: [
+ core
+ ]
params: {
- // TODO: Split dataFactory.bicep into its separate apps
- app: newApp(
- hub,
- 'Microsoft FinOps hubs',
- 'Microsoft.FinOpsHubs',
- 'DataFactory',
- 'FinOps hub engine',
- loadTextContent('ftkver.txt')
- )
-
- hubName: hubName
- dataFactoryName: core.outputs.dataFactoryName
- location: location
- tags: core.outputs.publisherTags
- tagsByResource: tagsByResource
- storageAccountName: core.outputs.storageAccountName
- exportContainerName: cmExports.outputs.exportContainer
- configContainerName: core.outputs.configContainer
- ingestionContainerName: core.outputs.ingestionContainer
- dataExplorerName: safeDataExplorerName
- dataExplorerPrincipalId: safeDataExplorerPrincipalId
- dataExplorerIngestionDatabase: safeDataExplorerIngestionDb
- dataExplorerIngestionCapacity: safeDataExplorerIngestionCapacity
- dataExplorerUri: safeDataExplorerUri
- dataExplorerId: safeDataExplorerId
- enableManagedExports: enableManagedExports
- enablePublicAccess: enablePublicAccess
-
- // TODO: Move to remoteHub.bicep
- keyVaultName: empty(remoteHubStorageKey) ? '' : remoteHub.outputs.keyVaultName
+ app: newApp(hub, 'Microsoft.FinOpsHubs', 'RemoteHub')
+ remoteStorageKey: remoteHubStorageKey
remoteHubStorageUri: remoteHubStorageUri
}
}
//------------------------------------------------------------------------------
-// Remote hub app
+// Final touches
//------------------------------------------------------------------------------
-module remoteHub 'remoteHub.bicep' = if (!empty(remoteHubStorageKey)) {
- name: 'Microsoft.FinOpsHubs.RemoteHub'
+// Delete old triggers and pipelines
+module deleteOldResources 'fx/hub-deploymentScript.bicep' = {
+ name: 'Microsoft.FinOpsHubs.DeleteOldResources'
params: {
- hub: hub
- remoteStorageKey: remoteHubStorageKey
+ app: core.outputs.app
+ identityName: core.outputs.triggerManagerIdentityName
+ scriptContent: loadTextContent('fx/scripts/Remove-OldResources.ps1')
+ environmentVariables: [
+ {
+ name: 'DataFactorySubscriptionId'
+ value: subscription().id
+ }
+ {
+ name: 'DataFactoryResourceGroup'
+ value: resourceGroup().name
+ }
+ {
+ name: 'DataFactoryName'
+ value: core.outputs.app.dataFactory
+ }
+ ]
+ }
+}
+
+// Start all ADF triggers
+module startTriggers 'fx/hub-initialize.bicep' = {
+ name: 'Microsoft.FinOpsHubs.StartTriggers'
+ params: {
+ app: core.outputs.app
+ dataFactoryInstances: [
+ core.outputs.app.dataFactory // Microsoft.FinOpsHubs
+ cmExports.outputs.app.dataFactory // Microsoft.CostManagement
+ ]
+ identityName: core.outputs.triggerManagerIdentityName
+ startAllTriggers: true
}
}
@@ -418,16 +383,20 @@ output storageAccountName string = core.outputs.storageAccountName
output storageUrlForPowerBI string = core.outputs.storageUrlForPowerBI
@description('The resource ID of the Data Explorer cluster.')
-output clusterId string = !deployDataExplorer ? '' : dataExplorer.outputs.clusterId
+#disable-next-line BCP318 // Null safety warning for conditional resource access
+output clusterId string = !useAzureDataExplorer ? '' : analytics.outputs.clusterId
@description('The URI of the Data Explorer cluster.')
-output clusterUri string = useFabric ? fabricQueryUri : (!deployDataExplorer ? '' : dataExplorer.outputs.clusterUri)
+#disable-next-line BCP318 // Null safety warning for conditional resource access
+output clusterUri string = useFabric ? fabricQueryUri : (!useAzureDataExplorer ? '' : analytics.outputs.clusterUri)
@description('The name of the Data Explorer database used for ingesting data.')
-output ingestionDbName string = useFabric ? 'Ingestion' : (!deployDataExplorer ? '' : dataExplorer.outputs.ingestionDbName)
+#disable-next-line BCP318 // Null safety warning for conditional resource access
+output ingestionDbName string = useFabric || useAzureDataExplorer ? analytics.outputs.ingestionDbName : ''
@description('The name of the Data Explorer database used for querying data.')
-output hubDbName string = useFabric ? 'Hub' : (!deployDataExplorer ? '' : dataExplorer.outputs.hubDbName)
+#disable-next-line BCP318 // Null safety warning for conditional resource access
+output hubDbName string = useFabric || useAzureDataExplorer ? analytics.outputs.hubDbName : ''
@description('Object ID of the Data Factory managed identity. This will be needed when configuring managed exports.')
output managedIdentityId string = core.outputs.principalId
diff --git a/src/templates/finops-hub/modules/remoteHub.bicep b/src/templates/finops-hub/modules/remoteHub.bicep
deleted file mode 100644
index 41175d497..000000000
--- a/src/templates/finops-hub/modules/remoteHub.bicep
+++ /dev/null
@@ -1,59 +0,0 @@
-// Copyright (c) Microsoft Corporation.
-// Licensed under the MIT License.
-
-import { HubProperties } from 'hub-types.bicep'
-
-
-//==============================================================================
-// Parameters
-//==============================================================================
-
-@description('Required. FinOps hub instance properties.')
-param hub HubProperties
-
-@description('Required. Create and store a key for a remote storage account.')
-@secure()
-param remoteStorageKey string
-
-
-//==============================================================================
-// Resources
-//==============================================================================
-
-// App registration
-module appRegistration 'hub-app.bicep' = {
- name: 'Microsoft.FinOpsHubs.RemoteHub_Register'
- params: {
- hub: hub
- publisher: 'Microsoft FinOps hubs'
- namespace: 'Microsoft.FinOpsHubs'
- appName: 'RemoteHub'
- displayName: 'FinOps hub remote relay'
- appVersion: loadTextContent('ftkver.txt') // cSpell:ignore ftkver
- features: [
- // TODO: Add pipeline -- 'DataFactory'
- 'KeyVault'
- 'Storage'
- ]
- }
-}
-
-// Key Vault secret
-module keyVault_secret 'hub-vault.bicep' = {
- name: 'keyVault_secret'
- params: {
- vaultName: appRegistration.outputs.app.keyVault
- secretName: '${toLower(appRegistration.outputs.app.hub.name)}-storage-key'
- secretValue: remoteStorageKey
- secretExpirationInSeconds: 1702648632
- secretNotBeforeInSeconds: 10000
- }
-}
-
-
-//==============================================================================
-// Outputs
-//==============================================================================
-
-@description('Name of the Key Vault instance.')
-output keyVaultName string = appRegistration.outputs.app.keyVault
diff --git a/src/templates/finops-hub/modules/scripts/Start-Triggers.ps1 b/src/templates/finops-hub/modules/scripts/Start-Triggers.ps1
deleted file mode 100644
index f462d811e..000000000
--- a/src/templates/finops-hub/modules/scripts/Start-Triggers.ps1
+++ /dev/null
@@ -1,65 +0,0 @@
-# Copyright (c) Microsoft Corporation.
-# Licensed under the MIT License.
-
-Param(
- [switch] $Stop
-)
-
-# Init outputs
-$DeploymentScriptOutputs = @{}
-
-if (-not $Stop)
-{
- Start-Sleep -Seconds 10
-}
-
-# Loop thru triggers
-$env:Triggers.Split('|') `
-| ForEach-Object {
- $trigger = $_
- if ($Stop)
- {
- Write-Output "Stopping trigger $trigger..."
- $triggerOutput = Stop-AzDataFactoryV2Trigger `
- -ResourceGroupName $env:DataFactoryResourceGroup `
- -DataFactoryName $env:DataFactoryName `
- -Name $trigger `
- -Force `
- -ErrorAction SilentlyContinue # Ignore errors, since the trigger may not exist
- }
- else
- {
- Write-Output "Starting trigger $trigger..."
- $triggerOutput = Start-AzDataFactoryV2Trigger `
- -ResourceGroupName $env:DataFactoryResourceGroup `
- -DataFactoryName $env:DataFactoryName `
- -Name $trigger `
- -Force
- }
- if ($triggerOutput)
- {
- Write-Output "done..."
- }
- else
- {
- Write-Output "failed..."
- }
- $DeploymentScriptOutputs[$trigger] = $triggerOutput
-}
-
-if ($Stop)
-{
- Start-Sleep -Seconds 10
-}
-
-if (-not [string]::IsNullOrWhiteSpace($env:Pipelines))
-{
- $env:Pipelines.Split('|') `
- | ForEach-Object {
- Write-Output "Running the init pipeline..."
- Invoke-AzDataFactoryV2Pipeline `
- -ResourceGroupName $env:DataFactoryResourceGroup `
- -DataFactoryName $env:DataFactoryName `
- -PipelineName $_
- }
-}
diff --git a/src/templates/finops-hub/schemas/README.md b/src/templates/finops-hub/schemas/README.md
deleted file mode 100644
index 346cf385c..000000000
--- a/src/templates/finops-hub/schemas/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-# 📦 FinOps hub schemas
-
-These schemas are used to indicate data types for the CSV to parquet conversion.
-
-- [focuscost_1.0.json](./focuscost_1.0.json)
-- [focuscost_1.0-preview(v1).json](./focuscost_1.0-preview(v1).json)
-
-