diff --git a/docs/_includes/version.txt b/docs/_includes/version.txt index 70e8e888c..0dd8f7c41 100644 --- a/docs/_includes/version.txt +++ b/docs/_includes/version.txt @@ -1 +1 @@ -0.2.1-rc.2 \ No newline at end of file +0.2.1-rc.3 \ No newline at end of file diff --git a/docs/changelog.md b/docs/changelog.md index 85e00617c..4709c043e 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -70,7 +70,10 @@ Legend: > ➕ Added: > -> 1. Started archiving template versions so they can be referenced easily via URL (microsoft.github.io/finops-toolkit/deploy/finops-hub-{version}.json). +> 1. Allow specifying an existing Key Vault instance. +> - If using template deployment, set the `existingKeyVaultId` parameter to the fully-qualified resource ID. +> - If using the `Deploy-FinOpsHub` PowerShell command, set the `-ExistingKeyVaultId` parameter to the fully-qualified resource ID. +> 2. Started archiving template versions so they can be referenced easily via URL (microsoft.github.io/finops-toolkit/deploy/finops-hub-{version}.json). > > 🛠️ Fixed: > diff --git a/docs/deploy/finops-hub-0.2.1-rc.3.json b/docs/deploy/finops-hub-0.2.1-rc.3.json new file mode 100644 index 000000000..0e32b0cd5 --- /dev/null +++ b/docs/deploy/finops-hub-0.2.1-rc.3.json @@ -0,0 +1,2138 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.24.24.22086", + "templateHash": "9748975281421053488" + } + }, + "parameters": { + "hubName": { + "type": "string", + "metadata": { + "description": "Optional. Name of the hub. Used to ensure unique resource names. Default: \"finops-hub\"." + } + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Optional. Azure location where all resources should be created. See https://aka.ms/azureregions. Default: Same as deployment." + } + }, + "storageSku": { + "type": "string", + "defaultValue": "Premium_LRS", + "allowedValues": [ + "Premium_LRS", + "Premium_ZRS" + ], + "metadata": { + "description": "Optional. Storage SKU to use. LRS = Lowest cost, ZRS = High availability. Note Standard SKUs are not available for Data Lake gen2 storage. Allowed: Premium_LRS, Premium_ZRS. Default: Premium_LRS." + } + }, + "existingKeyVaultId": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Optional. Resource ID of the existing Key Vault resource to use. If not specified, a new Key Vault instance will be created." + } + }, + "tags": { + "type": "object", + "defaultValue": {}, + "metadata": { + "description": "Optional. Tags to apply to all resources. We will also add the cm-resource-parent tag for improved cost roll-ups in Cost Management." + } + }, + "tagsByResource": { + "type": "object", + "defaultValue": {}, + "metadata": { + "description": "Optional. Tags to apply to resources based on their resource type. Resource type specific tags will be merged with tags for all resources." + } + }, + "exportScopes": { + "type": "array", + "defaultValue": [], + "metadata": { + "description": "Optional. List of scope IDs to create exports for." + } + } + }, + "resources": [ + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "hub", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "hubName": { + "value": "[parameters('hubName')]" + }, + "location": { + "value": "[parameters('location')]" + }, + "storageSku": { + "value": "[parameters('storageSku')]" + }, + "existingKeyVaultId": { + "value": "[parameters('existingKeyVaultId')]" + }, + "tags": { + "value": "[parameters('tags')]" + }, + "tagsByResource": { + "value": "[parameters('tagsByResource')]" + }, + "exportScopes": { + "value": "[parameters('exportScopes')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.24.24.22086", + "templateHash": "17411969862525330364" + } + }, + "parameters": { + "hubName": { + "type": "string", + "metadata": { + "description": "Optional. Name of the hub. Used to ensure unique resource names. Default: \"finops-hub\"." + } + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Optional. Azure location where all resources should be created. See https://aka.ms/azureregions. Default: (resource group location)." + } + }, + "storageSku": { + "type": "string", + "defaultValue": "Premium_LRS", + "allowedValues": [ + "Premium_LRS", + "Premium_ZRS" + ], + "metadata": { + "description": "Optional. Storage SKU to use. LRS = Lowest cost, ZRS = High availability. Note Standard SKUs are not available for Data Lake gen2 storage. Allowed: Premium_LRS, Premium_ZRS. Default: Premium_LRS." + } + }, + "existingKeyVaultId": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Optional. Resource ID of the existing Key Vault resource to use. If not specified, a new Key Vault instance will be created." + } + }, + "tags": { + "type": "object", + "defaultValue": {}, + "metadata": { + "description": "Optional. Tags to apply to all resources. We will also add the cm-resource-parent tag for improved cost roll-ups in Cost Management." + } + }, + "tagsByResource": { + "type": "object", + "defaultValue": {}, + "metadata": { + "description": "Optional. Tags to apply to resources based on their resource type. Resource type specific tags will be merged with tags for all resources." + } + }, + "exportScopes": { + "type": "array", + "metadata": { + "description": "Optional. List of scope IDs to create exports for." + } + }, + "convertToParquet": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Optional. Indicates whether ingested data should be converted to Parquet. Default: true." + } + }, + "enableDefaultTelemetry": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Optional. Enable telemetry to track anonymous module usage trends, monitor for bugs, and improve future releases." + } + } + }, + "variables": { + "resourceTags": "[union(parameters('tags'), createObject('cm-resource-parent', format('{0}/providers/Microsoft.Cloud/hubs/{1}', resourceGroup().id, parameters('hubName'))))]", + "uniqueSuffix": "[uniqueString(parameters('hubName'), resourceGroup().id)]", + "dataFactoryPrefix": "[format('{0}-engine', replace(parameters('hubName'), '_', '-'))]", + "dataFactorySuffix": "[format('-{0}', variables('uniqueSuffix'))]", + "dataFactoryName": "[replace(format('{0}{1}', take(variables('dataFactoryPrefix'), sub(63, length(variables('dataFactorySuffix')))), variables('dataFactorySuffix')), '--', '-')]", + "telemetryId": "00f120b5-2007-6120-0000-40b000000000", + "finOpsToolkitVersion": "placeholder" + }, + "resources": [ + { + "condition": "[parameters('enableDefaultTelemetry')]", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "[format('pid-{0}-{1}', variables('telemetryId'), uniqueString(deployment().name, parameters('location')))]", + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "FinOps toolkit", + "version": "[variables('finOpsToolkitVersion')]" + } + }, + "resources": [] + } + } + }, + { + "type": "Microsoft.DataFactory/factories", + "apiVersion": "2018-06-01", + "name": "[variables('dataFactoryName')]", + "location": "[parameters('location')]", + "tags": "[union(variables('resourceTags'), if(contains(parameters('tagsByResource'), 'Microsoft.DataFactory/factories'), parameters('tagsByResource')['Microsoft.DataFactory/factories'], createObject()))]", + "identity": { + "type": "SystemAssigned" + }, + "properties": "[union(createObject(), createObject('globalConfigurations', createObject('PipelineBillingEnabled', 'true')))]" + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "storage", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "hubName": { + "value": "[parameters('hubName')]" + }, + "uniqueSuffix": { + "value": "[variables('uniqueSuffix')]" + }, + "sku": { + "value": "[parameters('storageSku')]" + }, + "location": { + "value": "[parameters('location')]" + }, + "tags": { + "value": "[variables('resourceTags')]" + }, + "tagsByResource": { + "value": "[parameters('tagsByResource')]" + }, + "exportScopes": { + "value": "[parameters('exportScopes')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.24.24.22086", + "templateHash": "9609675916139055971" + } + }, + "parameters": { + "hubName": { + "type": "string", + "metadata": { + "description": "Required. Name of the hub. Used to ensure unique resource names." + } + }, + "uniqueSuffix": { + "type": "string", + "metadata": { + "description": "Required. Suffix to add to the storage account name to ensure uniqueness." + } + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Optional. Azure location where all resources should be created. See https://aka.ms/azureregions. Default: (resource group location)." + } + }, + "sku": { + "type": "string", + "defaultValue": "Premium_LRS", + "allowedValues": [ + "Premium_LRS", + "Premium_ZRS" + ], + "metadata": { + "description": "Optional. Storage SKU to use. LRS = Lowest cost, ZRS = High availability. Note Standard SKUs are not available for Data Lake gen2 storage. Allowed: Premium_LRS, Premium_ZRS. Default: Premium_LRS." + } + }, + "tags": { + "type": "object", + "defaultValue": {}, + "metadata": { + "description": "Optional. Tags to apply to all resources. We will also add the cm-resource-parent tag for improved cost roll-ups in Cost Management." + } + }, + "tagsByResource": { + "type": "object", + "defaultValue": {}, + "metadata": { + "description": "Optional. Tags to apply to resources based on their resource type. Resource type specific tags will be merged with tags for all resources." + } + }, + "exportScopes": { + "type": "array", + "metadata": { + "description": "Optional. List of scope IDs to create exports for." + } + } + }, + "variables": { + "$fxv#0": "placeholder", + "$fxv#1": "# Copyright (c) Microsoft Corporation.\r\n# Licensed under the MIT License.\r\n\r\nWrite-Output \"Updating settings.json file...\"\r\nWrite-Output \" Storage account: $env:storageAccountName\"\r\nWrite-Output \" Container: $env:containerName\"\r\n\r\n$validateScopes = { $_.Length -gt 45 }\r\n\r\n# Initialize variables\r\n$fileName = 'settings.json'\r\n$filePath = Join-Path -Path . -ChildPath $fileName\r\n$newScopes = $env:exportScopes.Split('|') | Where-Object $validateScopes | ForEach-Object { @{ scope = $_ } }\r\n\r\n# Get storage context\r\n$storageContext = @{\r\n Context = New-AzStorageContext -StorageAccountName $env:storageAccountName -UseConnectedAccount\r\n Container = $env:containerName\r\n}\r\n\r\n# Download existing settings, if they exist\r\n$blob = Get-AzStorageBlobContent @storageContext -Blob $fileName -Destination $filePath -Force\r\nif ($blob) {\r\n Write-Output \"Existing settings.json file found. Updating...\"\r\n $text = Get-Content $filePath -Raw\r\n Write-Output \"---------\"\r\n Write-Output $text\r\n Write-Output \"---------\"\r\n $json = $text | ConvertFrom-Json\r\n\r\n # Rename exportScopes to scopes + convert to object array\r\n if ($json.exportScopes) {\r\n Write-Output \" Updating exportScopes...\"\r\n if ($json.exportScopes[0] -is [string]) {\r\n Write-Output \" Converting string array to object array...\"\r\n $json.exportScopes = $json.exportScopes | Where-Object $validateScopes | ForEach-Object { @{ scope = $_ } }\r\n if (-not ($json.exportScopes -is [array])) {\r\n Write-Output \" Converting single object to object array...\"\r\n $json.exportScopes = @($json.exportScopes)\r\n }\r\n }\r\n\r\n Write-Output \" Renaming to 'scopes'...\"\r\n $json | Add-Member -MemberType NoteProperty -Name scopes -Value $json.exportScopes\r\n $json.PSObject.Properties.Remove('exportScopes')\r\n }\r\n}\r\n\r\n# Set default if not found\r\nif (!$json) {\r\n Write-Output \"No existing settings.json file found. Creating new file...\"\r\n $json = [ordered]@{\r\n '$schema' = 'https://aka.ms/finops/hubs/settings-schema'\r\n type = 'HubInstance'\r\n version = ''\r\n learnMore = 'https://aka.ms/finops/hubs'\r\n scopes = @()\r\n }\r\n}\r\n\r\n# Updating settings\r\nWrite-Output \"Updating version to $env:ftkVersion...\"\r\n$json.version = $env:ftkVersion\r\nif ($newScopes) {\r\n Write-Output \"Merging $($newScopes.Count) scopes...\"\r\n $json.scopes = Compare-Object -ReferenceObject $json.scopes -DifferenceObject $newScopes -Property scope -PassThru -IncludeEqual\r\n\r\n # Remove the SideIndicator property from the Compare-Object output\r\n $json.scopes | ForEach-Object { $_.PSObject.Properties.Remove('SideIndicator') } | ConvertTo-Json\r\n\r\n if (-not ($json.scopes -is [array])) {\r\n $json.scopes = @($json.scopes)\r\n }\r\n Write-Output \"$($json.scopes.Count) scopes found.\"\r\n}\r\n$text = $json | ConvertTo-Json\r\nWrite-Output \"---------\"\r\nWrite-Output $text\r\nWrite-Output \"---------\"\r\n$text | Out-File $filePath\r\n\r\n# Upload new/updated settings\r\nWrite-Output \"Uploading settings.json file...\"\r\nSet-AzStorageBlobContent @storageContext -File $filePath -Force\r\n", + "safeHubName": "[replace(replace(toLower(parameters('hubName')), '-', ''), '_', '')]", + "storageAccountSuffix": "[parameters('uniqueSuffix')]", + "storageAccountName": "[format('{0}{1}', take(variables('safeHubName'), sub(24, length(variables('storageAccountSuffix')))), variables('storageAccountSuffix'))]", + "blobUploadRbacRoles": [ + "ba92f5b4-2d11-453d-a403-e96b0029c9fe", + "e40ec5ca-96e0-45a2-b4ff-59039f2c2b59" + ] + }, + "resources": [ + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2021-08-01", + "name": "[variables('storageAccountName')]", + "location": "[parameters('location')]", + "sku": { + "name": "[parameters('sku')]" + }, + "kind": "BlockBlobStorage", + "tags": "[union(parameters('tags'), if(contains(parameters('tagsByResource'), 'Microsoft.Storage/storageAccounts'), parameters('tagsByResource')['Microsoft.Storage/storageAccounts'], createObject()))]", + "properties": { + "supportsHttpsTrafficOnly": true, + "isHnsEnabled": true, + "minimumTlsVersion": "TLS1_2", + "allowBlobPublicAccess": false + } + }, + { + "type": "Microsoft.Storage/storageAccounts/blobServices", + "apiVersion": "2021-06-01", + "name": "[format('{0}/{1}', variables('storageAccountName'), 'default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]" + ] + }, + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2021-06-01", + "name": "[format('{0}/{1}/{2}', variables('storageAccountName'), 'default', 'config')]", + "properties": { + "publicAccess": "None", + "metadata": {} + }, + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('storageAccountName'), 'default')]" + ] + }, + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2021-06-01", + "name": "[format('{0}/{1}/{2}', variables('storageAccountName'), 'default', 'msexports')]", + "properties": { + "publicAccess": "None", + "metadata": {} + }, + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('storageAccountName'), 'default')]" + ] + }, + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2021-06-01", + "name": "[format('{0}/{1}/{2}', variables('storageAccountName'), 'default', 'ingestion')]", + "properties": { + "publicAccess": "None", + "metadata": {} + }, + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('storageAccountName'), 'default')]" + ] + }, + { + "type": "Microsoft.ManagedIdentity/userAssignedIdentities", + "apiVersion": "2023-01-31", + "name": "[format('{0}_blobManager', variables('storageAccountName'))]", + "tags": "[union(parameters('tags'), if(contains(parameters('tagsByResource'), 'Microsoft.ManagedIdentity/userAssignedIdentities'), parameters('tagsByResource')['Microsoft.ManagedIdentity/userAssignedIdentities'], createObject()))]", + "location": "[parameters('location')]" + }, + { + "copy": { + "name": "identityRoleAssignments", + "count": "[length(variables('blobUploadRbacRoles'))]" + }, + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[guid(resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName')), variables('blobUploadRbacRoles')[copyIndex()], resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_blobManager', variables('storageAccountName'))))]", + "properties": { + "roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', variables('blobUploadRbacRoles')[copyIndex()])]", + "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_blobManager', variables('storageAccountName'))), '2023-01-31').principalId]", + "principalType": "ServicePrincipal" + }, + "dependsOn": [ + "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_blobManager', variables('storageAccountName')))]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]" + ] + }, + { + "type": "Microsoft.Resources/deploymentScripts", + "apiVersion": "2020-10-01", + "name": "uploadSettings", + "kind": "AzurePowerShell", + "location": "[if(startsWith(parameters('location'), 'china'), 'chinaeast2', parameters('location'))]", + "tags": "[union(parameters('tags'), if(contains(parameters('tagsByResource'), 'Microsoft.Resources/deploymentScripts'), parameters('tagsByResource')['Microsoft.Resources/deploymentScripts'], createObject()))]", + "identity": { + "type": "UserAssigned", + "userAssignedIdentities": { + "[format('{0}', resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_blobManager', variables('storageAccountName'))))]": {} + } + }, + "properties": { + "azPowerShellVersion": "8.0", + "retentionInterval": "PT1H", + "environmentVariables": [ + { + "name": "ftkVersion", + "value": "[variables('$fxv#0')]" + }, + { + "name": "exportScopes", + "value": "[join(parameters('exportScopes'), '|')]" + }, + { + "name": "storageAccountName", + "value": "[variables('storageAccountName')]" + }, + { + "name": "containerName", + "value": "config" + } + ], + "scriptContent": "[variables('$fxv#1')]" + }, + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices/containers', variables('storageAccountName'), 'default', 'config')]", + "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_blobManager', variables('storageAccountName')))]", + "identityRoleAssignments" + ] + } + ], + "outputs": { + "resourceId": { + "type": "string", + "metadata": { + "description": "The resource ID of the storage account." + }, + "value": "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]" + }, + "name": { + "type": "string", + "metadata": { + "description": "The name of the storage account." + }, + "value": "[variables('storageAccountName')]" + }, + "configContainer": { + "type": "string", + "metadata": { + "description": "The name of the container used for configuration settings." + }, + "value": "config" + }, + "exportContainer": { + "type": "string", + "metadata": { + "description": "The name of the container used for Cost Management exports." + }, + "value": "msexports" + }, + "ingestionContainer": { + "type": "string", + "metadata": { + "description": "The name of the container used for normalized data ingestion." + }, + "value": "ingestion" + } + } + } + } + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "dataFactoryResources", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "dataFactoryName": { + "value": "[variables('dataFactoryName')]" + }, + "convertToParquet": { + "value": "[parameters('convertToParquet')]" + }, + "keyVaultId": { + "value": "[reference(resourceId('Microsoft.Resources/deployments', 'keyVault'), '2022-09-01').outputs.resourceId.value]" + }, + "storageAccountName": { + "value": "[reference(resourceId('Microsoft.Resources/deployments', 'storage'), '2022-09-01').outputs.name.value]" + }, + "exportContainerName": { + "value": "[reference(resourceId('Microsoft.Resources/deployments', 'storage'), '2022-09-01').outputs.exportContainer.value]" + }, + "ingestionContainerName": { + "value": "[reference(resourceId('Microsoft.Resources/deployments', 'storage'), '2022-09-01').outputs.ingestionContainer.value]" + }, + "location": { + "value": "[parameters('location')]" + }, + "tags": { + "value": "[variables('resourceTags')]" + }, + "tagsByResource": { + "value": "[parameters('tagsByResource')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.24.24.22086", + "templateHash": "8197671316834274442" + } + }, + "parameters": { + "dataFactoryName": { + "type": "string", + "metadata": { + "description": "Required. Name of the hub. Used to ensure unique resource names." + } + }, + "keyVaultId": { + "type": "string", + "metadata": { + "description": "Optional. The resource ID of the Azure Key Vault instance." + } + }, + "storageAccountName": { + "type": "string", + "metadata": { + "description": "Required. The name of the Azure storage account instance." + } + }, + "exportContainerName": { + "type": "string", + "metadata": { + "description": "Required. The name of the container where Cost Management data is exported." + } + }, + "ingestionContainerName": { + "type": "string", + "metadata": { + "description": "Required. The name of the container where normalized data is ingested." + } + }, + "convertToParquet": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Optional. Indicates whether ingested data should be converted to Parquet. Default: true." + } + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Optional. The location to use for the managed identity and deployment script to auto-start triggers. Default = (resource group location)." + } + }, + "tags": { + "type": "object", + "defaultValue": {}, + "metadata": { + "description": "Optional. Tags to apply to all resources. We will also add the cm-resource-parent tag for improved cost roll-ups in Cost Management." + } + }, + "tagsByResource": { + "type": "object", + "defaultValue": {}, + "metadata": { + "description": "Optional. Tags to apply to resources based on their resource type. Resource type specific tags will be merged with tags for all resources." + } + } + }, + "variables": { + "copy": [ + { + "name": "focusCostMappings", + "count": "[length(range(0, length(variables('focusCostColumns'))))]", + "input": { + "source": { + "name": "[variables('focusCostColumns')[range(0, length(variables('focusCostColumns')))[copyIndex('focusCostMappings')]].name]", + "type": "[variables('focusCostColumns')[range(0, length(variables('focusCostColumns')))[copyIndex('focusCostMappings')]].type]" + }, + "sink": { + "name": "[variables('focusCostColumns')[range(0, length(variables('focusCostColumns')))[copyIndex('focusCostMappings')]].name]" + } + } + } + ], + "$fxv#0": "# Copyright (c) Microsoft Corporation.\r\n# Licensed under the MIT License.\r\n\r\n# Init outputs\r\n$DeploymentScriptOutputs = @{}\r\n\r\n# \r\n$adfParams = @{\r\n ResourceGroupName = $env:DataFactoryResourceGroup\r\n DataFactoryName = $env:DataFactoryName\r\n}\r\n\r\n# Delete old triggers\r\n$triggers = Get-AzDataFactoryV2Trigger @adfParams -ErrorAction SilentlyContinue `\r\n| Where-Object { $_.Name -match '^msexports?$' }\r\n$DeploymentScriptOutputs[\"stopTriggers\"] = $triggers | Stop-AzDataFactoryV2Trigger -Force -ErrorAction SilentlyContinue\r\n$DeploymentScriptOutputs[\"deleteTriggers\"] = $triggers | Remove-AzDataFactoryV2Trigger -Force -ErrorAction SilentlyContinue\r\n\r\n# Delete old pipelines\r\n$DeploymentScriptOutputs[\"pipelines\"] = Get-AzDataFactoryV2Pipeline @adfParams -ErrorAction SilentlyContinue `\r\n| Where-Object { $_.Name -match '^msexports_(extract|transform)$' } `\r\n| Remove-AzDataFactoryV2Pipeline -Force -ErrorAction SilentlyContinue", + "$fxv#1": "# Copyright (c) Microsoft Corporation.\r\n# Licensed under the MIT License.\r\n\r\nParam(\r\n [switch] $Stop\r\n)\r\n\r\n# Init outputs\r\n$DeploymentScriptOutputs = @{}\r\n\r\nif (-not $Stop) {\r\n Start-Sleep -Seconds 10\r\n}\r\n\r\n# Loop thru triggers\r\n$env:Triggers.Split('|') `\r\n| ForEach-Object {\r\n $trigger = $_\r\n if ($Stop) {\r\n Write-Host \"Stopping trigger $trigger...\" -NoNewline\r\n $triggerOutput = Stop-AzDataFactoryV2Trigger `\r\n -ResourceGroupName $env:DataFactoryResourceGroup `\r\n -DataFactoryName $env:DataFactoryName `\r\n -Name $trigger `\r\n -Force `\r\n -ErrorAction SilentlyContinue # Ignore errors, since the trigger may not exist\r\n } else {\r\n Write-Host \"Starting trigger $trigger...\" -NoNewline\r\n $triggerOutput = Start-AzDataFactoryV2Trigger `\r\n -ResourceGroupName $env:DataFactoryResourceGroup `\r\n -DataFactoryName $env:DataFactoryName `\r\n -Name $trigger `\r\n -Force\r\n }\r\n if ($triggerOutput) {\r\n Write-Host 'done'\r\n } else {\r\n Write-Host 'failed'\r\n }\r\n $DeploymentScriptOutputs[$trigger] = $triggerOutput\r\n}\r\n\r\nif ($Stop) {\r\n Start-Sleep -Seconds 10\r\n}\r\n", + "$fxv#2": "# Copyright (c) Microsoft Corporation.\r\n# Licensed under the MIT License.\r\n\r\nParam(\r\n [switch] $Stop\r\n)\r\n\r\n# Init outputs\r\n$DeploymentScriptOutputs = @{}\r\n\r\nif (-not $Stop) {\r\n Start-Sleep -Seconds 10\r\n}\r\n\r\n# Loop thru triggers\r\n$env:Triggers.Split('|') `\r\n| ForEach-Object {\r\n $trigger = $_\r\n if ($Stop) {\r\n Write-Host \"Stopping trigger $trigger...\" -NoNewline\r\n $triggerOutput = Stop-AzDataFactoryV2Trigger `\r\n -ResourceGroupName $env:DataFactoryResourceGroup `\r\n -DataFactoryName $env:DataFactoryName `\r\n -Name $trigger `\r\n -Force `\r\n -ErrorAction SilentlyContinue # Ignore errors, since the trigger may not exist\r\n } else {\r\n Write-Host \"Starting trigger $trigger...\" -NoNewline\r\n $triggerOutput = Start-AzDataFactoryV2Trigger `\r\n -ResourceGroupName $env:DataFactoryResourceGroup `\r\n -DataFactoryName $env:DataFactoryName `\r\n -Name $trigger `\r\n -Force\r\n }\r\n if ($triggerOutput) {\r\n Write-Host 'done'\r\n } else {\r\n Write-Host 'failed'\r\n }\r\n $DeploymentScriptOutputs[$trigger] = $triggerOutput\r\n}\r\n\r\nif ($Stop) {\r\n Start-Sleep -Seconds 10\r\n}\r\n", + "datasetPropsDelimitedText": { + "columnDelimiter": ",", + "compressionLevel": "Optimal", + "escapeChar": "\"", + "firstRowAsHeader": true, + "quoteChar": "\"" + }, + "datasetPropsCommon": { + "location": { + "type": "AzureBlobFSLocation", + "fileName": { + "value": "@{dataset().fileName}", + "type": "Expression" + }, + "folderPath": { + "value": "@{dataset().folderName}", + "type": "Expression" + } + } + }, + "safeExportContainerName": "[replace(format('{0}', parameters('exportContainerName')), '-', '_')]", + "safeIngestionContainerName": "[replace(format('{0}', parameters('ingestionContainerName')), '-', '_')]", + "exportFileAddedTriggerName": "[format('{0}_FileAdded', variables('safeExportContainerName'))]", + "allHubTriggers": [ + "[variables('exportFileAddedTriggerName')]" + ], + "autoStartRbacRoles": [ + "673868aa-7521-48a0-acc6-0f60742d39f5", + "e40ec5ca-96e0-45a2-b4ff-59039f2c2b59" + ], + "focusCostColumns": [ + { + "name": "AvailabilityZone", + "type": "String" + }, + { + "name": "BilledCost", + "type": "Decimal" + }, + { + "name": "BillingAccountId", + "type": "String" + }, + { + "name": "BillingAccountName", + "type": "String" + }, + { + "name": "BillingAccountType", + "type": "String" + }, + { + "name": "BillingCurrency", + "type": "String" + }, + { + "name": "BillingPeriodEnd", + "type": "DateTime" + }, + { + "name": "BillingPeriodStart", + "type": "DateTime" + }, + { + "name": "ChargeCategory", + "type": "String" + }, + { + "name": "ChargeDescription", + "type": "String" + }, + { + "name": "ChargeFrequency", + "type": "String" + }, + { + "name": "ChargePeriodEnd", + "type": "DateTime" + }, + { + "name": "ChargePeriodStart", + "type": "DateTime" + }, + { + "name": "ChargeSubcategory", + "type": "String" + }, + { + "name": "CommitmentDiscountCategory", + "type": "String" + }, + { + "name": "CommitmentDiscountId", + "type": "String" + }, + { + "name": "CommitmentDiscountName", + "type": "String" + }, + { + "name": "CommitmentDiscountType", + "type": "String" + }, + { + "name": "EffectiveCost", + "type": "Decimal" + }, + { + "name": "InvoiceIssuerName", + "type": "String" + }, + { + "name": "ListCost", + "type": "Decimal" + }, + { + "name": "ListUnitPrice", + "type": "Decimal" + }, + { + "name": "PricingCategory", + "type": "String" + }, + { + "name": "PricingQuantity", + "type": "Decimal" + }, + { + "name": "PricingUnit", + "type": "String" + }, + { + "name": "ProviderName", + "type": "String" + }, + { + "name": "PublisherName", + "type": "String" + }, + { + "name": "Region", + "type": "String" + }, + { + "name": "ResourceId", + "type": "String" + }, + { + "name": "ResourceName", + "type": "String" + }, + { + "name": "ResourceType", + "type": "String" + }, + { + "name": "ServiceCategory", + "type": "String" + }, + { + "name": "ServiceName", + "type": "String" + }, + { + "name": "SkuId", + "type": "String" + }, + { + "name": "SkuPriceId", + "type": "String" + }, + { + "name": "SubAccountId", + "type": "String" + }, + { + "name": "SubAccountName", + "type": "String" + }, + { + "name": "SubAccountType", + "type": "String" + }, + { + "name": "Tags", + "type": "String" + }, + { + "name": "UsageQuantity", + "type": "Decimal" + }, + { + "name": "UsageUnit", + "type": "String" + }, + { + "name": "x_AccountName", + "type": "String" + }, + { + "name": "x_AccountOwnerId", + "type": "String" + }, + { + "name": "x_BilledCostInUsd", + "type": "Decimal" + }, + { + "name": "x_BilledUnitPrice", + "type": "Decimal" + }, + { + "name": "x_BillingAccountId", + "type": "String" + }, + { + "name": "x_BillingAccountName", + "type": "String" + }, + { + "name": "x_BillingExchangeRate", + "type": "Decimal" + }, + { + "name": "x_BillingExchangeRateDate", + "type": "DateTime" + }, + { + "name": "x_BillingProfileId", + "type": "String" + }, + { + "name": "x_BillingProfileName", + "type": "String" + }, + { + "name": "x_ChargeId", + "type": "String" + }, + { + "name": "x_CostAllocationRuleName", + "type": "String" + }, + { + "name": "x_CostCenter", + "type": "String" + }, + { + "name": "x_CustomerId", + "type": "String" + }, + { + "name": "x_CustomerName", + "type": "String" + }, + { + "name": "x_EffectiveCostInUsd", + "type": "Decimal" + }, + { + "name": "x_EffectiveUnitPrice", + "type": "Decimal" + }, + { + "name": "x_InvoiceId", + "type": "String" + }, + { + "name": "x_InvoiceIssuerId", + "type": "String" + }, + { + "name": "x_InvoiceSectionId", + "type": "String" + }, + { + "name": "x_InvoiceSectionName", + "type": "String" + }, + { + "name": "x_OnDemandCost", + "type": "Decimal" + }, + { + "name": "x_OnDemandCostInUsd", + "type": "Decimal" + }, + { + "name": "x_OnDemandUnitPrice", + "type": "Decimal" + }, + { + "name": "x_PartnerCreditApplied", + "type": "Boolean" + }, + { + "name": "x_PartnerCreditRate", + "type": "Decimal" + }, + { + "name": "x_PricingBlockSize", + "type": "Decimal" + }, + { + "name": "x_PricingCurrency", + "type": "String" + }, + { + "name": "x_PricingSubcategory", + "type": "String" + }, + { + "name": "x_PricingUnitDescription", + "type": "String" + }, + { + "name": "x_PublisherCategory", + "type": "String" + }, + { + "name": "x_PublisherId", + "type": "String" + }, + { + "name": "x_ResellerId", + "type": "String" + }, + { + "name": "x_ResellerName", + "type": "String" + }, + { + "name": "x_ResourceGroupName", + "type": "String" + }, + { + "name": "x_ResourceType", + "type": "String" + }, + { + "name": "x_ServicePeriodEnd", + "type": "DateTime" + }, + { + "name": "x_ServicePeriodStart", + "type": "DateTime" + }, + { + "name": "x_SkuDescription", + "type": "String" + }, + { + "name": "x_SkuDetails", + "type": "String" + }, + { + "name": "x_SkuIsCreditEligible", + "type": "Boolean" + }, + { + "name": "x_SkuMeterCategory", + "type": "String" + }, + { + "name": "x_SkuMeterId", + "type": "String" + }, + { + "name": "x_SkuMeterName", + "type": "String" + }, + { + "name": "x_SkuMeterSubcategory", + "type": "String" + }, + { + "name": "x_SkuOfferId", + "type": "String" + }, + { + "name": "x_SkuOrderId", + "type": "String" + }, + { + "name": "x_SkuOrderName", + "type": "String" + }, + { + "name": "x_SkuPartNumber", + "type": "String" + }, + { + "name": "x_SkuRegion", + "type": "String" + }, + { + "name": "x_SkuServiceFamily", + "type": "String" + }, + { + "name": "x_SkuTerm", + "type": "String" + }, + { + "name": "x_SkuTier", + "type": "String" + } + ] + }, + "resources": [ + { + "type": "Microsoft.Resources/deploymentScripts", + "apiVersion": "2020-10-01", + "name": "[format('{0}_deleteOldResources', parameters('dataFactoryName'))]", + "location": "[parameters('location')]", + "identity": { + "type": "UserAssigned", + "userAssignedIdentities": { + "[format('{0}', resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_triggerManager', parameters('dataFactoryName'))))]": {} + } + }, + "kind": "AzurePowerShell", + "tags": "[parameters('tags')]", + "properties": { + "azPowerShellVersion": "8.0", + "retentionInterval": "PT1H", + "cleanupPreference": "OnSuccess", + "scriptContent": "[variables('$fxv#0')]", + "environmentVariables": [ + { + "name": "DataFactorySubscriptionId", + "value": "[subscription().id]" + }, + { + "name": "DataFactoryResourceGroup", + "value": "[resourceGroup().name]" + }, + { + "name": "DataFactoryName", + "value": "[parameters('dataFactoryName')]" + } + ] + }, + "dependsOn": [ + "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_triggerManager', parameters('dataFactoryName')))]", + "identityRoleAssignments" + ] + }, + { + "type": "Microsoft.ManagedIdentity/userAssignedIdentities", + "apiVersion": "2023-01-31", + "name": "[format('{0}_triggerManager', parameters('dataFactoryName'))]", + "location": "[parameters('location')]", + "tags": "[union(parameters('tags'), if(contains(parameters('tagsByResource'), 'Microsoft.ManagedIdentity/userAssignedIdentities'), parameters('tagsByResource')['Microsoft.ManagedIdentity/userAssignedIdentities'], createObject()))]" + }, + { + "copy": { + "name": "identityRoleAssignments", + "count": "[length(variables('autoStartRbacRoles'))]" + }, + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "scope": "[format('Microsoft.DataFactory/factories/{0}', parameters('dataFactoryName'))]", + "name": "[guid(resourceId('Microsoft.DataFactory/factories', parameters('dataFactoryName')), variables('autoStartRbacRoles')[copyIndex()], resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_triggerManager', parameters('dataFactoryName'))))]", + "properties": { + "roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', variables('autoStartRbacRoles')[copyIndex()])]", + "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_triggerManager', parameters('dataFactoryName'))), '2023-01-31').principalId]", + "principalType": "ServicePrincipal" + }, + "dependsOn": [ + "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_triggerManager', parameters('dataFactoryName')))]" + ] + }, + { + "type": "Microsoft.Resources/deploymentScripts", + "apiVersion": "2020-10-01", + "name": "[format('{0}_stopHubTriggers', parameters('dataFactoryName'))]", + "location": "[if(startsWith(parameters('location'), 'china'), 'chinaeast2', parameters('location'))]", + "identity": { + "type": "UserAssigned", + "userAssignedIdentities": { + "[format('{0}', resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_triggerManager', parameters('dataFactoryName'))))]": {} + } + }, + "kind": "AzurePowerShell", + "tags": "[union(parameters('tags'), if(contains(parameters('tagsByResource'), 'Microsoft.Resources/deploymentScripts'), parameters('tagsByResource')['Microsoft.Resources/deploymentScripts'], createObject()))]", + "properties": { + "azPowerShellVersion": "8.0", + "retentionInterval": "PT1H", + "cleanupPreference": "OnSuccess", + "scriptContent": "[variables('$fxv#1')]", + "arguments": "-Stop", + "environmentVariables": [ + { + "name": "DataFactorySubscriptionId", + "value": "[subscription().id]" + }, + { + "name": "DataFactoryResourceGroup", + "value": "[resourceGroup().name]" + }, + { + "name": "DataFactoryName", + "value": "[parameters('dataFactoryName')]" + }, + { + "name": "Triggers", + "value": "[join(variables('allHubTriggers'), '|')]" + } + ] + }, + "dependsOn": [ + "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_triggerManager', parameters('dataFactoryName')))]", + "identityRoleAssignments" + ] + }, + { + "type": "Microsoft.DataFactory/factories/linkedservices", + "apiVersion": "2018-06-01", + "name": "[format('{0}/{1}', parameters('dataFactoryName'), 'keyVault')]", + "properties": { + "annotations": [], + "parameters": {}, + "type": "AzureKeyVault", + "typeProperties": { + "baseUrl": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', split(parameters('keyVaultId'), '/')[2], split(parameters('keyVaultId'), '/')[4]), 'Microsoft.KeyVault/vaults', last(split(parameters('keyVaultId'), '/'))), '2022-11-01').vaultUri]" + } + } + }, + { + "type": "Microsoft.DataFactory/factories/linkedservices", + "apiVersion": "2018-06-01", + "name": "[format('{0}/{1}', parameters('dataFactoryName'), 'storage')]", + "properties": { + "annotations": [], + "parameters": {}, + "type": "AzureBlobFS", + "typeProperties": { + "url": "[reference(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2022-09-01').primaryEndpoints.dfs]", + "accountKey": { + "type": "AzureKeyVaultSecret", + "store": { + "referenceName": "keyVault", + "type": "LinkedServiceReference" + }, + "secretName": "[parameters('storageAccountName')]" + } + } + }, + "dependsOn": [ + "[resourceId('Microsoft.DataFactory/factories/linkedservices', parameters('dataFactoryName'), 'keyVault')]" + ] + }, + { + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "name": "[format('{0}/{1}', parameters('dataFactoryName'), variables('safeExportContainerName'))]", + "properties": { + "annotations": [], + "parameters": { + "fileName": { + "type": "String" + }, + "folderName": { + "type": "String" + } + }, + "type": "DelimitedText", + "typeProperties": "[union(variables('datasetPropsCommon'), variables('datasetPropsDelimitedText'), createObject('compressionCodec', 'none'))]", + "linkedServiceName": { + "parameters": {}, + "referenceName": "storage", + "type": "LinkedServiceReference" + } + }, + "dependsOn": [ + "[resourceId('Microsoft.DataFactory/factories/linkedservices', parameters('dataFactoryName'), 'keyVault')]", + "[resourceId('Microsoft.DataFactory/factories/linkedservices', parameters('dataFactoryName'), 'storage')]" + ] + }, + { + "type": "Microsoft.DataFactory/factories/datasets", + "apiVersion": "2018-06-01", + "name": "[format('{0}/{1}', parameters('dataFactoryName'), variables('safeIngestionContainerName'))]", + "properties": { + "annotations": [], + "parameters": { + "fileName": { + "type": "String" + }, + "folderName": { + "type": "String" + } + }, + "type": "[if(parameters('convertToParquet'), 'Parquet', 'DelimitedText')]", + "typeProperties": "[union(variables('datasetPropsCommon'), if(parameters('convertToParquet'), createObject(), variables('datasetPropsDelimitedText')), createObject('compressionCodec', 'gzip'))]", + "linkedServiceName": { + "parameters": {}, + "referenceName": "storage", + "type": "LinkedServiceReference" + } + }, + "dependsOn": [ + "[resourceId('Microsoft.DataFactory/factories/linkedservices', parameters('dataFactoryName'), 'keyVault')]", + "[resourceId('Microsoft.DataFactory/factories/linkedservices', parameters('dataFactoryName'), 'storage')]" + ] + }, + { + "type": "Microsoft.DataFactory/factories/triggers", + "apiVersion": "2018-06-01", + "name": "[format('{0}/{1}', parameters('dataFactoryName'), variables('exportFileAddedTriggerName'))]", + "properties": { + "annotations": [], + "pipelines": [ + { + "pipelineReference": { + "referenceName": "[format('{0}_ExecuteETL', parameters('exportContainerName'))]", + "type": "PipelineReference" + }, + "parameters": { + "folderName": "@triggerBody().folderPath", + "fileName": "@triggerBody().fileName" + } + } + ], + "type": "BlobEventsTrigger", + "typeProperties": { + "blobPathBeginsWith": "[format('/{0}/blobs/', parameters('exportContainerName'))]", + "blobPathEndsWith": ".csv", + "ignoreEmptyBlobs": true, + "scope": "[resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName'))]", + "events": [ + "Microsoft.Storage.BlobCreated" + ] + } + }, + "dependsOn": [ + "[resourceId('Microsoft.DataFactory/factories/pipelines', parameters('dataFactoryName'), format('{0}_ExecuteETL', variables('safeExportContainerName')))]", + "[resourceId('Microsoft.Resources/deploymentScripts', format('{0}_stopHubTriggers', parameters('dataFactoryName')))]" + ] + }, + { + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "name": "[format('{0}/{1}', parameters('dataFactoryName'), format('{0}_ExecuteETL', variables('safeExportContainerName')))]", + "properties": { + "activities": [ + { + "name": "Execute", + "type": "ExecutePipeline", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "pipeline": { + "referenceName": "[format('{0}_ETL_{1}', variables('safeExportContainerName'), variables('safeIngestionContainerName'))]", + "type": "PipelineReference" + }, + "waitOnCompletion": false, + "parameters": { + "folderName": { + "value": "@pipeline().parameters.folderName", + "type": "Expression" + }, + "fileName": { + "value": "@pipeline().parameters.fileName", + "type": "Expression" + } + } + } + } + ], + "parameters": { + "folderName": { + "type": "string" + }, + "fileName": { + "type": "string" + } + }, + "annotations": [] + }, + "dependsOn": [ + "[resourceId('Microsoft.DataFactory/factories/pipelines', parameters('dataFactoryName'), format('{0}_ETL_{1}', variables('safeExportContainerName'), variables('safeIngestionContainerName')))]" + ] + }, + { + "type": "Microsoft.DataFactory/factories/pipelines", + "apiVersion": "2018-06-01", + "name": "[format('{0}/{1}', parameters('dataFactoryName'), format('{0}_ETL_{1}', variables('safeExportContainerName'), variables('safeIngestionContainerName')))]", + "properties": { + "activities": [ + { + "name": "Wait", + "type": "Wait", + "dependsOn": [], + "userProperties": [], + "typeProperties": { + "waitTimeInSeconds": 60 + } + }, + { + "name": "Set FolderArray", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Wait", + "dependencyConditions": [ + "Completed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "variableName": "folderArray", + "value": { + "value": "@split(pipeline().parameters.folderName, '/')", + "type": "Expression" + } + } + }, + { + "name": "Set FolderCount", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Set FolderArray", + "dependencyConditions": [ + "Completed" + ] + } + ], + "policy": { + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "variableName": "folderCount", + "value": "@length(split(pipeline().parameters.folderName, '/'))" + } + }, + { + "name": "Set SecondToLastFolder", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Set FolderCount", + "dependencyConditions": [ + "Completed" + ] + } + ], + "policy": { + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "variableName": "secondToLastFolder", + "value": "@variables('folderArray')[sub(variables('folderCount'), 2)]" + } + }, + { + "name": "Set ThirdToLastFolder", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Set SecondToLastFolder", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "variableName": "thirdToLastFolder", + "value": "@variables('folderArray')[sub(variables('folderCount'), 3)]" + } + }, + { + "name": "Set FourthToLastFolder", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Set ThirdToLastFolder", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "variableName": "fourthToLastFolder", + "value": "@variables('folderArray')[sub(variables('folderCount'), 4)]" + } + }, + { + "name": "Set Scope", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Set FourthToLastFolder", + "dependencyConditions": [ + "Completed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "variableName": "scope", + "value": { + "value": "[format('@replace(split(pipeline().parameters.folderName, if(greater(length(variables(''secondToLastFolder'')), 12), variables(''thirdToLastFolder''), variables(''fourthToLastFolder'')))[0], ''{0}'', ''{1}'')', parameters('exportContainerName'), parameters('ingestionContainerName'))]", + "type": "Expression" + } + } + }, + { + "name": "Set Metric", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Set Scope", + "dependencyConditions": [ + "Completed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "variableName": "metric", + "value": { + "value": "focuscost", + "type": "Expression" + } + } + }, + { + "name": "Set Date", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Set Metric", + "dependencyConditions": [ + "Completed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "variableName": "date", + "value": { + "value": "@substring(if(greater(length(variables('secondToLastFolder')), 12), variables('secondToLastFolder'), variables('thirdToLastFolder')), 0, 6)", + "type": "Expression" + } + } + }, + { + "name": "Set Destination File Name", + "description": "", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Set Date", + "dependencyConditions": [ + "Completed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "variableName": "destinationFile", + "value": { + "value": "[format('@replace(pipeline().parameters.fileName, ''.csv'', ''{0}'')', if(parameters('convertToParquet'), '.parquet', '.csv.gz'))]", + "type": "Expression" + } + } + }, + { + "name": "Set Destination Folder Name", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Set Destination File Name", + "dependencyConditions": [ + "Completed" + ] + } + ], + "userProperties": [], + "typeProperties": { + "variableName": "destinationFolder", + "value": { + "value": "@replace(concat(variables('scope'),variables('date'),'/',variables('metric')),'//','/')", + "type": "Expression" + } + } + }, + { + "name": "Delete Target", + "type": "Delete", + "dependsOn": [ + { + "activity": "Set Destination Folder Name", + "dependencyConditions": [ + "Completed" + ] + } + ], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "dataset": { + "referenceName": "[variables('safeIngestionContainerName')]", + "type": "DatasetReference", + "parameters": { + "folderName": { + "value": "@variables('destinationFolder')", + "type": "Expression" + }, + "fileName": { + "value": "@variables('destinationFile')", + "type": "Expression" + } + } + }, + "enableLogging": false, + "storeSettings": { + "type": "AzureBlobFSReadSettings", + "recursive": true, + "enablePartitionDiscovery": false + } + } + }, + { + "name": "Convert CSV", + "type": "Copy", + "dependsOn": [ + { + "activity": "Delete Target", + "dependencyConditions": [ + "Completed" + ] + } + ], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "source": { + "type": "DelimitedTextSource", + "storeSettings": { + "type": "AzureBlobFSReadSettings", + "recursive": true, + "enablePartitionDiscovery": false + }, + "formatSettings": { + "type": "DelimitedTextReadSettings" + } + }, + "sink": { + "type": "DelimitedTextSink", + "storeSettings": { + "type": "AzureBlobFSWriteSettings" + }, + "formatSettings": "[if(parameters('convertToParquet'), createObject('type', 'ParquetWriteSettings', 'fileExtension', '.parquet'), createObject('type', 'DelimitedTextWriteSettings', 'quoteAllText', true(), 'fileExtension', '.csv.gz'))]" + }, + "enableStaging": false, + "parallelCopies": 1, + "validateDataConsistency": false, + "translator": { + "type": "TabularTranslator", + "mappings": "[variables('focusCostMappings')]" + } + }, + "inputs": [ + { + "referenceName": "[variables('safeExportContainerName')]", + "type": "DatasetReference", + "parameters": { + "folderName": { + "value": "@pipeline().parameters.folderName", + "type": "Expression" + }, + "fileName": { + "value": "@pipeline().parameters.fileName", + "type": "Expression" + } + } + } + ], + "outputs": [ + { + "referenceName": "[variables('safeIngestionContainerName')]", + "type": "DatasetReference", + "parameters": { + "folderName": { + "value": "@variables('destinationFolder')", + "type": "Expression" + }, + "fileName": { + "value": "@variables('destinationFile')", + "type": "Expression" + } + } + } + ] + }, + { + "name": "Delete CSV", + "type": "Delete", + "dependsOn": [ + { + "activity": "Convert CSV", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "userProperties": [], + "typeProperties": { + "dataset": { + "referenceName": "[variables('safeExportContainerName')]", + "type": "DatasetReference", + "parameters": { + "folderName": { + "value": "@pipeline().parameters.folderName", + "type": "Expression" + }, + "fileName": { + "value": "@pipeline().parameters.fileName", + "type": "Expression" + } + } + }, + "enableLogging": false, + "storeSettings": { + "type": "AzureBlobFSReadSettings", + "recursive": true, + "enablePartitionDiscovery": false + } + } + } + ], + "parameters": { + "fileName": { + "type": "string" + }, + "folderName": { + "type": "string" + } + }, + "variables": { + "destinationFile": { + "type": "String" + }, + "destinationFolder": { + "type": "String" + }, + "folderArray": { + "type": "Array" + }, + "folderCount": { + "type": "Integer" + }, + "secondToLastFolder": { + "type": "String" + }, + "thirdToLastFolder": { + "type": "String" + }, + "fourthToLastFolder": { + "type": "String" + }, + "scope": { + "type": "String" + }, + "date": { + "type": "String" + }, + "metric": { + "type": "String" + } + }, + "annotations": [] + }, + "dependsOn": [ + "[resourceId('Microsoft.DataFactory/factories/datasets', parameters('dataFactoryName'), variables('safeIngestionContainerName'))]", + "[resourceId('Microsoft.DataFactory/factories/datasets', parameters('dataFactoryName'), variables('safeExportContainerName'))]" + ] + }, + { + "type": "Microsoft.Resources/deploymentScripts", + "apiVersion": "2020-10-01", + "name": "[format('{0}_startHubTriggers', parameters('dataFactoryName'))]", + "location": "[if(startsWith(parameters('location'), 'china'), 'chinaeast2', parameters('location'))]", + "tags": "[union(parameters('tags'), if(contains(parameters('tagsByResource'), 'Microsoft.Resources/deploymentScripts'), parameters('tagsByResource')['Microsoft.Resources/deploymentScripts'], createObject()))]", + "identity": { + "type": "UserAssigned", + "userAssignedIdentities": { + "[format('{0}', resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_triggerManager', parameters('dataFactoryName'))))]": {} + } + }, + "kind": "AzurePowerShell", + "properties": { + "azPowerShellVersion": "8.0", + "retentionInterval": "PT1H", + "cleanupPreference": "OnSuccess", + "scriptContent": "[variables('$fxv#2')]", + "environmentVariables": [ + { + "name": "DataFactorySubscriptionId", + "value": "[subscription().id]" + }, + { + "name": "DataFactoryResourceGroup", + "value": "[resourceGroup().name]" + }, + { + "name": "DataFactoryName", + "value": "[parameters('dataFactoryName')]" + }, + { + "name": "Triggers", + "value": "[join(variables('allHubTriggers'), '|')]" + } + ] + }, + "dependsOn": [ + "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}_triggerManager', parameters('dataFactoryName')))]", + "identityRoleAssignments", + "[resourceId('Microsoft.DataFactory/factories/triggers', parameters('dataFactoryName'), variables('exportFileAddedTriggerName'))]" + ] + } + ], + "outputs": { + "resourceId": { + "type": "string", + "metadata": { + "description": "The Resource ID of the Data factory." + }, + "value": "[resourceId('Microsoft.DataFactory/factories', parameters('dataFactoryName'))]" + }, + "name": { + "type": "string", + "metadata": { + "description": "The Name of the Azure Data Factory instance." + }, + "value": "[parameters('dataFactoryName')]" + } + } + } + }, + "dependsOn": [ + "[resourceId('Microsoft.Resources/deployments', 'keyVault')]", + "[resourceId('Microsoft.Resources/deployments', 'storage')]" + ] + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "keyVault", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "hubName": { + "value": "[parameters('hubName')]" + }, + "existingKeyVaultName": { + "value": "[last(split(parameters('existingKeyVaultId'), '/'))]" + }, + "uniqueSuffix": { + "value": "[variables('uniqueSuffix')]" + }, + "location": { + "value": "[parameters('location')]" + }, + "tags": { + "value": "[variables('resourceTags')]" + }, + "tagsByResource": { + "value": "[parameters('tagsByResource')]" + }, + "storageAccountName": { + "value": "[reference(resourceId('Microsoft.Resources/deployments', 'storage'), '2022-09-01').outputs.name.value]" + }, + "accessPolicies": { + "value": [ + { + "objectId": "[reference(resourceId('Microsoft.DataFactory/factories', variables('dataFactoryName')), '2018-06-01', 'full').identity.principalId]", + "tenantId": "[subscription().tenantId]", + "permissions": { + "secrets": [ + "get" + ] + } + } + ] + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.24.24.22086", + "templateHash": "18385434849803379988" + } + }, + "parameters": { + "hubName": { + "type": "string", + "metadata": { + "description": "Required. Name of the hub. Used to ensure unique resource names." + } + }, + "uniqueSuffix": { + "type": "string", + "metadata": { + "description": "Required. Suffix to add to the KeyVault instance name to ensure uniqueness." + } + }, + "existingKeyVaultName": { + "type": "string", + "metadata": { + "description": "Optional. Resource ID of the existing Key Vault resource to use. If not specified, a new Key Vault instance will be created." + } + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Optional. Location for all resources." + } + }, + "accessPolicies": { + "type": "array", + "defaultValue": [], + "metadata": { + "description": "Optional. Array of access policies object." + } + }, + "storageAccountName": { + "type": "string", + "metadata": { + "description": "Required. Name of the storage account to store access keys for." + } + }, + "sku": { + "type": "string", + "defaultValue": "premium", + "allowedValues": [ + "premium", + "standard" + ], + "metadata": { + "description": "Optional. Specifies the SKU for the vault." + } + }, + "tags": { + "type": "object", + "defaultValue": {}, + "metadata": { + "description": "Optional. Resource tags." + } + }, + "tagsByResource": { + "type": "object", + "defaultValue": {}, + "metadata": { + "description": "Optional. Tags to apply to resources based on their resource type. Resource type specific tags will be merged with tags for all resources." + } + } + }, + "variables": { + "copy": [ + { + "name": "formattedAccessPolicies", + "count": "[length(parameters('accessPolicies'))]", + "input": { + "applicationId": "[if(contains(parameters('accessPolicies')[copyIndex('formattedAccessPolicies')], 'applicationId'), parameters('accessPolicies')[copyIndex('formattedAccessPolicies')].applicationId, '')]", + "objectId": "[if(contains(parameters('accessPolicies')[copyIndex('formattedAccessPolicies')], 'objectId'), parameters('accessPolicies')[copyIndex('formattedAccessPolicies')].objectId, '')]", + "permissions": "[parameters('accessPolicies')[copyIndex('formattedAccessPolicies')].permissions]", + "tenantId": "[if(contains(parameters('accessPolicies')[copyIndex('formattedAccessPolicies')], 'tenantId'), parameters('accessPolicies')[copyIndex('formattedAccessPolicies')].tenantId, tenant().tenantId)]" + } + } + ], + "keyVaultPrefix": "[format('{0}-vault', replace(parameters('hubName'), '_', '-'))]", + "keyVaultSuffix": "[format('-{0}', parameters('uniqueSuffix'))]", + "keyVaultName": "[replace(format('{0}{1}', take(variables('keyVaultPrefix'), sub(24, length(variables('keyVaultSuffix')))), variables('keyVaultSuffix')), '--', '-')]" + }, + "resources": [ + { + "condition": "[and(not(empty(parameters('existingKeyVaultName'))), not(empty(parameters('accessPolicies'))))]", + "type": "Microsoft.KeyVault/vaults/accessPolicies", + "apiVersion": "2023-07-01", + "name": "[format('{0}/{1}', if(empty(parameters('existingKeyVaultName')), 'placeholder', parameters('existingKeyVaultName')), 'add')]", + "properties": { + "accessPolicies": "[variables('formattedAccessPolicies')]" + } + }, + { + "condition": "[not(empty(parameters('existingKeyVaultName')))]", + "type": "Microsoft.KeyVault/vaults/secrets", + "apiVersion": "2023-07-01", + "name": "[format('{0}/{1}', if(empty(parameters('existingKeyVaultName')), 'placeholder', parameters('existingKeyVaultName')), parameters('storageAccountName'))]", + "properties": { + "attributes": { + "enabled": true, + "exp": 1702648632, + "nbf": 10000 + }, + "value": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2022-09-01').keys[0].value]" + } + }, + { + "condition": "[and(empty(parameters('existingKeyVaultName')), not(empty(parameters('accessPolicies'))))]", + "type": "Microsoft.KeyVault/vaults/accessPolicies", + "apiVersion": "2023-07-01", + "name": "[format('{0}/{1}', variables('keyVaultName'), 'add')]", + "properties": { + "accessPolicies": "[variables('formattedAccessPolicies')]" + }, + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]" + ] + }, + { + "condition": "[empty(parameters('existingKeyVaultName'))]", + "type": "Microsoft.KeyVault/vaults/secrets", + "apiVersion": "2023-07-01", + "name": "[format('{0}/{1}', variables('keyVaultName'), parameters('storageAccountName'))]", + "properties": { + "attributes": { + "enabled": true, + "exp": 1702648632, + "nbf": 10000 + }, + "value": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2022-09-01').keys[0].value]" + }, + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]" + ] + }, + { + "condition": "[empty(parameters('existingKeyVaultName'))]", + "type": "Microsoft.KeyVault/vaults", + "apiVersion": "2023-07-01", + "name": "[variables('keyVaultName')]", + "location": "[parameters('location')]", + "tags": "[union(parameters('tags'), if(contains(parameters('tagsByResource'), 'Microsoft.KeyVault/vaults'), parameters('tagsByResource')['Microsoft.KeyVault/vaults'], createObject()))]", + "properties": { + "enabledForDeployment": true, + "enabledForTemplateDeployment": true, + "enabledForDiskEncryption": true, + "enableSoftDelete": true, + "softDeleteRetentionInDays": 90, + "enableRbacAuthorization": false, + "createMode": "default", + "tenantId": "[subscription().tenantId]", + "accessPolicies": "[variables('formattedAccessPolicies')]", + "sku": { + "name": "[if(startsWith(parameters('location'), 'china'), 'standard', parameters('sku'))]", + "family": "A" + } + } + } + ], + "outputs": { + "resourceId": { + "type": "string", + "metadata": { + "description": "The resource ID of the key vault." + }, + "value": "[if(empty(parameters('existingKeyVaultName')), resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName')), resourceId('Microsoft.KeyVault/vaults', if(empty(parameters('existingKeyVaultName')), 'placeholder', parameters('existingKeyVaultName'))))]" + }, + "name": { + "type": "string", + "metadata": { + "description": "The name of the key vault." + }, + "value": "[if(empty(parameters('existingKeyVaultName')), variables('keyVaultName'), if(empty(parameters('existingKeyVaultName')), 'placeholder', parameters('existingKeyVaultName')))]" + }, + "uri": { + "type": "string", + "metadata": { + "description": "The URI of the key vault." + }, + "value": "[if(empty(parameters('existingKeyVaultName')), reference(resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName')), '2023-07-01').vaultUri, reference(resourceId('Microsoft.KeyVault/vaults', if(empty(parameters('existingKeyVaultName')), 'placeholder', parameters('existingKeyVaultName'))), '2023-07-01').vaultUri)]" + } + } + } + }, + "dependsOn": [ + "[resourceId('Microsoft.DataFactory/factories', variables('dataFactoryName'))]", + "[resourceId('Microsoft.Resources/deployments', 'storage')]" + ] + } + ], + "outputs": { + "name": { + "type": "string", + "metadata": { + "description": "Name of the deployed hub instance." + }, + "value": "[parameters('hubName')]" + }, + "location": { + "type": "string", + "metadata": { + "description": "Azure resource location resources were deployed to." + }, + "value": "[parameters('location')]" + }, + "dataFactorytName": { + "type": "string", + "metadata": { + "description": "Name of the Data Factory." + }, + "value": "[variables('dataFactoryName')]" + }, + "storageAccountId": { + "type": "string", + "metadata": { + "description": "Resource ID of the storage account created for the hub instance. This must be used when creating the Cost Management export." + }, + "value": "[reference(resourceId('Microsoft.Resources/deployments', 'storage'), '2022-09-01').outputs.resourceId.value]" + }, + "storageAccountName": { + "type": "string", + "metadata": { + "description": "Name of the storage account created for the hub instance. This must be used when connecting FinOps toolkit Power BI reports to your data." + }, + "value": "[reference(resourceId('Microsoft.Resources/deployments', 'storage'), '2022-09-01').outputs.name.value]" + }, + "storageUrlForPowerBI": { + "type": "string", + "metadata": { + "description": "URL to use when connecting custom Power BI reports to your data." + }, + "value": "[format('https://{0}.dfs.{1}/{2}', reference(resourceId('Microsoft.Resources/deployments', 'storage'), '2022-09-01').outputs.name.value, environment().suffixes.storage, reference(resourceId('Microsoft.Resources/deployments', 'storage'), '2022-09-01').outputs.ingestionContainer.value)]" + } + } + } + } + } + ], + "outputs": { + "name": { + "type": "string", + "metadata": { + "description": "The name of the resource group." + }, + "value": "[parameters('hubName')]" + }, + "location": { + "type": "string", + "metadata": { + "description": "The location the resources wer deployed to." + }, + "value": "[parameters('location')]" + }, + "dataFactorytName": { + "type": "string", + "metadata": { + "description": "Name of the Data Factory." + }, + "value": "[reference(resourceId('Microsoft.Resources/deployments', 'hub'), '2022-09-01').outputs.dataFactorytName.value]" + }, + "storageAccountId": { + "type": "string", + "metadata": { + "description": "The resource ID of the deployed storage account." + }, + "value": "[reference(resourceId('Microsoft.Resources/deployments', 'hub'), '2022-09-01').outputs.storageAccountId.value]" + }, + "storageAccountName": { + "type": "string", + "metadata": { + "description": "Name of the storage account created for the hub instance. This must be used when connecting FinOps toolkit Power BI reports to your data." + }, + "value": "[reference(resourceId('Microsoft.Resources/deployments', 'hub'), '2022-09-01').outputs.storageAccountName.value]" + }, + "storageUrlForPowerBI": { + "type": "string", + "metadata": { + "description": "URL to use when connecting custom Power BI reports to your data." + }, + "value": "[reference(resourceId('Microsoft.Resources/deployments', 'hub'), '2022-09-01').outputs.storageUrlForPowerBI.value]" + } + } +} \ No newline at end of file diff --git a/docs/deploy/finops-hub-latest.json b/docs/deploy/finops-hub-latest.json index 929aef644..0e32b0cd5 100644 --- a/docs/deploy/finops-hub-latest.json +++ b/docs/deploy/finops-hub-latest.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.24.24.22086", - "templateHash": "6371350577264419703" + "templateHash": "9748975281421053488" } }, "parameters": { @@ -33,6 +33,13 @@ "description": "Optional. Storage SKU to use. LRS = Lowest cost, ZRS = High availability. Note Standard SKUs are not available for Data Lake gen2 storage. Allowed: Premium_LRS, Premium_ZRS. Default: Premium_LRS." } }, + "existingKeyVaultId": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Optional. Resource ID of the existing Key Vault resource to use. If not specified, a new Key Vault instance will be created." + } + }, "tags": { "type": "object", "defaultValue": {}, @@ -75,6 +82,9 @@ "storageSku": { "value": "[parameters('storageSku')]" }, + "existingKeyVaultId": { + "value": "[parameters('existingKeyVaultId')]" + }, "tags": { "value": "[parameters('tags')]" }, @@ -92,7 +102,7 @@ "_generator": { "name": "bicep", "version": "0.24.24.22086", - "templateHash": "2610829918662778812" + "templateHash": "17411969862525330364" } }, "parameters": { @@ -120,6 +130,13 @@ "description": "Optional. Storage SKU to use. LRS = Lowest cost, ZRS = High availability. Note Standard SKUs are not available for Data Lake gen2 storage. Allowed: Premium_LRS, Premium_ZRS. Default: Premium_LRS." } }, + "existingKeyVaultId": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Optional. Resource ID of the existing Key Vault resource to use. If not specified, a new Key Vault instance will be created." + } + }, "tags": { "type": "object", "defaultValue": {}, @@ -487,8 +504,8 @@ "convertToParquet": { "value": "[parameters('convertToParquet')]" }, - "keyVaultName": { - "value": "[reference(resourceId('Microsoft.Resources/deployments', 'keyVault'), '2022-09-01').outputs.name.value]" + "keyVaultId": { + "value": "[reference(resourceId('Microsoft.Resources/deployments', 'keyVault'), '2022-09-01').outputs.resourceId.value]" }, "storageAccountName": { "value": "[reference(resourceId('Microsoft.Resources/deployments', 'storage'), '2022-09-01').outputs.name.value]" @@ -516,20 +533,20 @@ "_generator": { "name": "bicep", "version": "0.24.24.22086", - "templateHash": "5738194981634133446" + "templateHash": "8197671316834274442" } }, "parameters": { "dataFactoryName": { "type": "string", "metadata": { - "description": "Optional. Name of the hub. Used to ensure unique resource names. Default: \"finops-hub\"." + "description": "Required. Name of the hub. Used to ensure unique resource names." } }, - "keyVaultName": { + "keyVaultId": { "type": "string", "metadata": { - "description": "Required. The name of the Azure Key Vault instance." + "description": "Optional. The resource ID of the Azure Key Vault instance." } }, "storageAccountName": { @@ -1123,7 +1140,7 @@ "parameters": {}, "type": "AzureKeyVault", "typeProperties": { - "baseUrl": "[reference(resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName')), '2022-11-01').vaultUri]" + "baseUrl": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', split(parameters('keyVaultId'), '/')[2], split(parameters('keyVaultId'), '/')[4]), 'Microsoft.KeyVault/vaults', last(split(parameters('keyVaultId'), '/'))), '2022-11-01').vaultUri]" } } }, @@ -1794,6 +1811,9 @@ "hubName": { "value": "[parameters('hubName')]" }, + "existingKeyVaultName": { + "value": "[last(split(parameters('existingKeyVaultId'), '/'))]" + }, "uniqueSuffix": { "value": "[variables('uniqueSuffix')]" }, @@ -1830,7 +1850,7 @@ "_generator": { "name": "bicep", "version": "0.24.24.22086", - "templateHash": "10770478197596540923" + "templateHash": "18385434849803379988" } }, "parameters": { @@ -1846,6 +1866,12 @@ "description": "Required. Suffix to add to the KeyVault instance name to ensure uniqueness." } }, + "existingKeyVaultName": { + "type": "string", + "metadata": { + "description": "Optional. Resource ID of the existing Key Vault resource to use. If not specified, a new Key Vault instance will be created." + } + }, "location": { "type": "string", "defaultValue": "[resourceGroup().location]", @@ -1911,31 +1937,32 @@ }, "resources": [ { - "type": "Microsoft.KeyVault/vaults", - "apiVersion": "2022-11-01", - "name": "[variables('keyVaultName')]", - "location": "[parameters('location')]", - "tags": "[union(parameters('tags'), if(contains(parameters('tagsByResource'), 'Microsoft.KeyVault/vaults'), parameters('tagsByResource')['Microsoft.KeyVault/vaults'], createObject()))]", + "condition": "[and(not(empty(parameters('existingKeyVaultName'))), not(empty(parameters('accessPolicies'))))]", + "type": "Microsoft.KeyVault/vaults/accessPolicies", + "apiVersion": "2023-07-01", + "name": "[format('{0}/{1}', if(empty(parameters('existingKeyVaultName')), 'placeholder', parameters('existingKeyVaultName')), 'add')]", "properties": { - "enabledForDeployment": true, - "enabledForTemplateDeployment": true, - "enabledForDiskEncryption": true, - "enableSoftDelete": true, - "softDeleteRetentionInDays": 90, - "enableRbacAuthorization": false, - "createMode": "default", - "tenantId": "[subscription().tenantId]", - "accessPolicies": "[variables('formattedAccessPolicies')]", - "sku": { - "name": "[if(startsWith(parameters('location'), 'china'), 'standard', parameters('sku'))]", - "family": "A" - } + "accessPolicies": "[variables('formattedAccessPolicies')]" } }, { - "condition": "[not(empty(parameters('accessPolicies')))]", + "condition": "[not(empty(parameters('existingKeyVaultName')))]", + "type": "Microsoft.KeyVault/vaults/secrets", + "apiVersion": "2023-07-01", + "name": "[format('{0}/{1}', if(empty(parameters('existingKeyVaultName')), 'placeholder', parameters('existingKeyVaultName')), parameters('storageAccountName'))]", + "properties": { + "attributes": { + "enabled": true, + "exp": 1702648632, + "nbf": 10000 + }, + "value": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2022-09-01').keys[0].value]" + } + }, + { + "condition": "[and(empty(parameters('existingKeyVaultName')), not(empty(parameters('accessPolicies'))))]", "type": "Microsoft.KeyVault/vaults/accessPolicies", - "apiVersion": "2022-11-01", + "apiVersion": "2023-07-01", "name": "[format('{0}/{1}', variables('keyVaultName'), 'add')]", "properties": { "accessPolicies": "[variables('formattedAccessPolicies')]" @@ -1945,8 +1972,9 @@ ] }, { + "condition": "[empty(parameters('existingKeyVaultName'))]", "type": "Microsoft.KeyVault/vaults/secrets", - "apiVersion": "2022-11-01", + "apiVersion": "2023-07-01", "name": "[format('{0}/{1}', variables('keyVaultName'), parameters('storageAccountName'))]", "properties": { "attributes": { @@ -1959,6 +1987,29 @@ "dependsOn": [ "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]" ] + }, + { + "condition": "[empty(parameters('existingKeyVaultName'))]", + "type": "Microsoft.KeyVault/vaults", + "apiVersion": "2023-07-01", + "name": "[variables('keyVaultName')]", + "location": "[parameters('location')]", + "tags": "[union(parameters('tags'), if(contains(parameters('tagsByResource'), 'Microsoft.KeyVault/vaults'), parameters('tagsByResource')['Microsoft.KeyVault/vaults'], createObject()))]", + "properties": { + "enabledForDeployment": true, + "enabledForTemplateDeployment": true, + "enabledForDiskEncryption": true, + "enableSoftDelete": true, + "softDeleteRetentionInDays": 90, + "enableRbacAuthorization": false, + "createMode": "default", + "tenantId": "[subscription().tenantId]", + "accessPolicies": "[variables('formattedAccessPolicies')]", + "sku": { + "name": "[if(startsWith(parameters('location'), 'china'), 'standard', parameters('sku'))]", + "family": "A" + } + } } ], "outputs": { @@ -1967,21 +2018,21 @@ "metadata": { "description": "The resource ID of the key vault." }, - "value": "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]" + "value": "[if(empty(parameters('existingKeyVaultName')), resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName')), resourceId('Microsoft.KeyVault/vaults', if(empty(parameters('existingKeyVaultName')), 'placeholder', parameters('existingKeyVaultName'))))]" }, "name": { "type": "string", "metadata": { "description": "The name of the key vault." }, - "value": "[variables('keyVaultName')]" + "value": "[if(empty(parameters('existingKeyVaultName')), variables('keyVaultName'), if(empty(parameters('existingKeyVaultName')), 'placeholder', parameters('existingKeyVaultName')))]" }, "uri": { "type": "string", "metadata": { "description": "The URI of the key vault." }, - "value": "[reference(resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName')), '2022-11-01').vaultUri]" + "value": "[if(empty(parameters('existingKeyVaultName')), reference(resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName')), '2023-07-01').vaultUri, reference(resourceId('Microsoft.KeyVault/vaults', if(empty(parameters('existingKeyVaultName')), 'placeholder', parameters('existingKeyVaultName'))), '2023-07-01').vaultUri)]" } } } diff --git a/docs/finops-hub/template.md b/docs/finops-hub/template.md index 27ea2c67b..42db2e9b4 100644 --- a/docs/finops-hub/template.md +++ b/docs/finops-hub/template.md @@ -72,13 +72,14 @@ Please ensure the following prerequisites are met before deploying this template ## 📥 Parameters -| Parameter | Type | Description | Default value | -| ---------------- | ------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- | -| **hubName** | String | Optional. Name of the hub. Used to ensure unique resource names. | `"finops-hub"` | -| **location** | String | Optional. Azure location where all resources should be created. See https://aka.ms/azureregions. | (resource group location) | -| **storageSku** | String | Optional. Storage SKU to use. LRS = Lowest cost, ZRS = High availability. Note Standard SKUs are not available for Data Lake gen2 storage. Allowed: `Premium_LRS`, `Premium_ZRS`. | `Premium_LRS` | -| **tags** | Object | Optional. Tags to apply to all resources. We will also add the `cm-resource-parent` tag for improved cost roll-ups in Cost Management. | -| **exportScopes** | Array | Optional. List of scope IDs to create exports for. | +| Parameter | Type | Description | Default value | +| ---------------------- | ------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- | +| **hubName** | String | Optional. Name of the hub. Used to ensure unique resource names. | `"finops-hub"` | +| **location** | String | Optional. Azure location where all resources should be created. See https://aka.ms/azureregions. | (resource group location) | +| **storageSku** | String | Optional. Storage SKU to use. LRS = Lowest cost, ZRS = High availability. Note Standard SKUs are not available for Data Lake gen2 storage. Allowed: `Premium_LRS`, `Premium_ZRS`. | `Premium_LRS` | +| **existingKeyVaultId** | String | Optional. Resource ID of the existing Key Vault resource to use. If not specified, a new Key Vault instance will be created. | | +| **tags** | Object | Optional. Tags to apply to all resources. We will also add the `cm-resource-parent` tag for improved cost roll-ups in Cost Management. | | +| **exportScopes** | Array | Optional. List of scope IDs to create exports for. | |
diff --git a/docs/powershell/hubs/Deploy-FinOpsHub.md b/docs/powershell/hubs/Deploy-FinOpsHub.md index bbf374662..62dccb0bf 100644 --- a/docs/powershell/hubs/Deploy-FinOpsHub.md +++ b/docs/powershell/hubs/Deploy-FinOpsHub.md @@ -40,6 +40,7 @@ Deploy-FinOpsHub ` -Name ` -ResourceGroup ` -Location ` + [-KeyVaultId ] ` [-Version ] ` [-Preview] ` [-StorageSku ] ` @@ -57,6 +58,7 @@ Deploy-FinOpsHub ` | `‑ResourceGroup` | Required. Name of the resource group to deploy to. Will be created if it doesn't exist. | | `‑Location` | Required. Azure location to execute the deployment from. | | `‑Version` | Optional. Version of the FinOps hub template to use. Default = "latest". | +| `‑KeyVaultId` | Optional. Resource ID of the existing Key Vault instance to use. If not specified, one will be created. | | `‑Preview` | Optional. Indicates that preview releases should also be included. Default = false. | | `‑StorageSku` | Optional. Storage account SKU. Premium_LRS = Lowest cost, Premium_ZRS = High availability. Note Standard SKUs are not available for Data Lake gen2 storage. Default = "Premium_LRS". | | `‑Tags` | Optional. Tags for all resources. | @@ -88,12 +90,24 @@ Deploy-FinOpsHub ` Deploys a new FinOps hub instance named MyHub to a new resource group named MyNewResourceGroup using version {% include version.txt %} of the template. +### Use existing Key Vault instance + +```powershell +Deploy-FinOpsHub ` + -Name MyHub ` + -ResourceGroupName MyExistingResourceGroup ` + -Location westus ` + -KeyVaultId "/subscriptions/###/resourceGroups/###/providers/Microsoft.KeyVault/vaults/foo" +``` + +Deploys a new FinOps hub instance named MyHub using an existing Key Vault instance. +
--- ## 🧰 Related tools -{% include tools.md hubs="1" %} +{% include tools.md hubs="1" pbi="1" %}
diff --git a/package-lock.json b/package-lock.json index 9bb2855d9..77390a755 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "ftk", - "version": "0.2.1-rc.2", + "version": "0.2.1-rc.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "ftk", - "version": "0.2.1-rc.2", + "version": "0.2.1-rc.3", "license": "MIT", "devDependencies": { "all-contributors-cli": "^6.26.1" diff --git a/package.json b/package.json index e3a395a58..54fd8d075 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ftk", - "version": "0.2.1-rc.2", + "version": "0.2.1-rc.3", "description": "Starter kits, scripts, and advanced solutions to accelerate your FinOps journey in the Microsoft Cloud.", "main": "index.js", "directories": { diff --git a/src/powershell/Public/Deploy-FinOpsHub.ps1 b/src/powershell/Public/Deploy-FinOpsHub.ps1 index 6384a006f..74677bca3 100644 --- a/src/powershell/Public/Deploy-FinOpsHub.ps1 +++ b/src/powershell/Public/Deploy-FinOpsHub.ps1 @@ -19,6 +19,9 @@ .PARAMETER Location Required. Azure location to execute the deployment from. + .PARAMETER KeyVaultId + Optional. Resource ID of the existing Key Vault instance to use. If not specified, one will be created. + .PARAMETER Version Optional. Version of the FinOps hub template to use. Default = "latest". @@ -35,12 +38,17 @@ Deploy-FinOpsHub -Name MyHub -ResourceGroupName MyExistingResourceGroup -Location westus Deploys a new FinOps hub instance named MyHub to an existing resource group named MyExistingResourceGroup. - + .EXAMPLE Deploy-FinOpsHub -Name MyHub -Location westus -Version 0.1 - + Deploys a new FinOps hub instance named MyHub using version 0.1 of the template. + .EXAMPLE + Deploy-FinOpsHub -Name MyHub -ResourceGroupName MyExistingResourceGroup -Location westus -KeyVaultId "/subscriptions/###/resourceGroups/###/providers/Microsoft.KeyVault/vaults/foo" + + Deploys a new FinOps hub instance named MyHub using an existing Key Vault instance. + .LINK https://aka.ms/ftk/Deploy-FinOpsHub #> @@ -62,6 +70,10 @@ function Deploy-FinOpsHub [string] $Location, + [Parameter(Mandatory = $false)] + [string] + $KeyVaultId = '', + [Parameter()] [string] $Version = 'latest', @@ -112,6 +124,7 @@ function Deploy-FinOpsHub TemplateParameterObject = @{ hubName = $Name storageSku = $StorageSku + existingKeyVaultId = $KeyVaultId } } diff --git a/src/scripts/Package-Toolkit.ps1 b/src/scripts/Package-Toolkit.ps1 index af3ab6f05..e622bad8b 100644 --- a/src/scripts/Package-Toolkit.ps1 +++ b/src/scripts/Package-Toolkit.ps1 @@ -57,12 +57,12 @@ if ($Template -ne "*" -and -not (Test-Path $relDir)) return } -Write-Host "Packaging v$version templates..." - # Package templates $version = & "$PSScriptRoot/Get-Version" $isPrerelease = $version -like '*-*' +Write-Host "Packaging v$version templates..." + Write-Verbose "Removing existing ZIP files..." Remove-Item "$relDir/*.zip" -Force @@ -93,9 +93,9 @@ $templates = Get-ChildItem $relDir -Directory ` } # Copy azuredeploy.json to docs/deploy folder - if ($isPrerelease -and -not $Docs) + if ($Docs -or -not $isPrerelease) { - Write-Verbose "Updating $($path.Name) deployment file in docs..." + Write-Verbose "Updating $($path.Name) deployment files in docs..." Copy-Item "$path/azuredeploy.json" "$deployDir/$($path.Name)-$version.json" Copy-Item "$path/azuredeploy.json" "$deployDir/$($path.Name)-latest.json" } @@ -105,7 +105,10 @@ $templates = Get-ChildItem $relDir -Directory ` return $zip } Write-Host "✅ $($templates.Count) templates" -Write-Host "ℹ️ Deployment files updated... Please commit the changes manually..." +if ($Docs -or -not $isPrerelease) +{ + Write-Host "ℹ️ Deployment files updated... Please commit the changes manually..." +} # Copy open data files Write-Verbose "Copying open data files..." diff --git a/src/templates/finops-hub/main.bicep b/src/templates/finops-hub/main.bicep index b26b1888e..f45efa423 100644 --- a/src/templates/finops-hub/main.bicep +++ b/src/templates/finops-hub/main.bicep @@ -20,6 +20,9 @@ param location string = resourceGroup().location @description('Optional. Storage SKU to use. LRS = Lowest cost, ZRS = High availability. Note Standard SKUs are not available for Data Lake gen2 storage. Allowed: Premium_LRS, Premium_ZRS. Default: Premium_LRS.') param storageSku string = 'Premium_LRS' +@description('Optional. Resource ID of the existing Key Vault resource to use. If not specified, a new Key Vault instance will be created.') +param existingKeyVaultId string = '' + @description('Optional. Tags to apply to all resources. We will also add the cm-resource-parent tag for improved cost roll-ups in Cost Management.') param tags object = {} @@ -39,6 +42,7 @@ module hub 'modules/hub.bicep' = { hubName: hubName location: location storageSku: storageSku + existingKeyVaultId: existingKeyVaultId tags: tags tagsByResource: tagsByResource exportScopes: exportScopes diff --git a/src/templates/finops-hub/modules/dataFactory.bicep b/src/templates/finops-hub/modules/dataFactory.bicep index c03387fa7..946b37ce5 100644 --- a/src/templates/finops-hub/modules/dataFactory.bicep +++ b/src/templates/finops-hub/modules/dataFactory.bicep @@ -5,11 +5,11 @@ // Parameters //============================================================================== -@description('Optional. Name of the hub. Used to ensure unique resource names. Default: "finops-hub".') +@description('Required. Name of the hub. Used to ensure unique resource names.') param dataFactoryName string -@description('Required. The name of the Azure Key Vault instance.') -param keyVaultName string +@description('Optional. The resource ID of the Azure Key Vault instance.') +param keyVaultId string @description('Required. The name of the Azure storage account instance.') param storageAccountName string @@ -293,7 +293,8 @@ resource stopHubTriggers 'Microsoft.Resources/deploymentScripts@2020-10-01' = { //------------------------------------------------------------------------------ resource keyVault 'Microsoft.KeyVault/vaults@2022-11-01' existing = { - name: keyVaultName + name: last(split(keyVaultId, '/')) + scope: resourceGroup(split(keyVaultId, '/')[2], split(keyVaultId, '/')[4]) } resource linkedService_keyVault 'Microsoft.DataFactory/factories/linkedservices@2018-06-01' = { diff --git a/src/templates/finops-hub/modules/hub.bicep b/src/templates/finops-hub/modules/hub.bicep index f6226e421..2c6e8f17e 100644 --- a/src/templates/finops-hub/modules/hub.bicep +++ b/src/templates/finops-hub/modules/hub.bicep @@ -18,6 +18,9 @@ param location string = resourceGroup().location @description('Optional. Storage SKU to use. LRS = Lowest cost, ZRS = High availability. Note Standard SKUs are not available for Data Lake gen2 storage. Allowed: Premium_LRS, Premium_ZRS. Default: Premium_LRS.') param storageSku string = 'Premium_LRS' +@description('Optional. Resource ID of the existing Key Vault resource to use. If not specified, a new Key Vault instance will be created.') +param existingKeyVaultId string = '' + @description('Optional. Tags to apply to all resources. We will also add the cm-resource-parent tag for improved cost roll-ups in Cost Management.') param tags object = {} @@ -122,7 +125,7 @@ module dataFactoryResources 'dataFactory.bicep' = { params: { dataFactoryName: dataFactoryName convertToParquet: convertToParquet - keyVaultName: keyVault.outputs.name + keyVaultId: keyVault.outputs.resourceId storageAccountName: storage.outputs.name exportContainerName: storage.outputs.exportContainer ingestionContainerName: storage.outputs.ingestionContainer @@ -138,8 +141,10 @@ module dataFactoryResources 'dataFactory.bicep' = { module keyVault 'keyVault.bicep' = { name: 'keyVault' + scope: empty(existingKeyVaultId) ? resourceGroup() : resourceGroup(split(existingKeyVaultId, '/')[2], split(existingKeyVaultId, '/')[4]) params: { hubName: hubName + existingKeyVaultName: last(split(existingKeyVaultId, '/')) uniqueSuffix: uniqueSuffix location: location tags: resourceTags diff --git a/src/templates/finops-hub/modules/keyVault.bicep b/src/templates/finops-hub/modules/keyVault.bicep index 719839f6e..78166d522 100644 --- a/src/templates/finops-hub/modules/keyVault.bicep +++ b/src/templates/finops-hub/modules/keyVault.bicep @@ -11,6 +11,9 @@ param hubName string @description('Required. Suffix to add to the KeyVault instance name to ensure uniqueness.') param uniqueSuffix string +@description('Optional. Resource ID of the existing Key Vault resource to use. If not specified, a new Key Vault instance will be created.') +param existingKeyVaultName string + @description('Optional. Location for all resources.') param location string = resourceGroup().location @@ -49,11 +52,42 @@ var formattedAccessPolicies = [for accessPolicy in accessPolicies: { tenantId: contains(accessPolicy, 'tenantId') ? accessPolicy.tenantId : tenant().tenantId }] +var storageSecretProperties = { + attributes: { + enabled: true + exp: 1702648632 + nbf: 10000 + } + value: storageRef.listKeys().keys[0].value +} + //============================================================================== // Resources //============================================================================== -resource keyVault 'Microsoft.KeyVault/vaults@2022-11-01' = { +resource storageRef 'Microsoft.Storage/storageAccounts@2022-09-01' existing = { + name: storageAccountName +} + +// Get existing key vault, if existingKeyVaultName is set +resource existingKeyVault 'Microsoft.KeyVault/vaults@2023-07-01' existing = if (!empty(existingKeyVaultName)) { + name: empty(existingKeyVaultName) ? 'placeholder' : existingKeyVaultName + + resource existingKeyVault_accessPolicies 'accessPolicies@2023-07-01' = if (!empty(accessPolicies)) { + name: 'add' + properties: { + accessPolicies: formattedAccessPolicies + } + } + + resource existingKeyVault_secrets 'secrets@2023-07-01' = { + name: storageRef.name + properties: storageSecretProperties + } +} + +// Create new key vault, if existingKeyVaultName is not set +resource keyVault 'Microsoft.KeyVault/vaults@2023-07-01' = if (empty(existingKeyVaultName)) { name: keyVaultName location: location tags: union(tags, contains(tagsByResource, 'Microsoft.KeyVault/vaults') ? tagsByResource['Microsoft.KeyVault/vaults'] : {}) @@ -73,30 +107,17 @@ resource keyVault 'Microsoft.KeyVault/vaults@2022-11-01' = { family: 'A' } } -} - -resource keyVault_accessPolicies 'Microsoft.KeyVault/vaults/accessPolicies@2022-11-01' = if (!empty(accessPolicies)) { - name: 'add' - parent: keyVault - properties: { - accessPolicies: formattedAccessPolicies - } -} -resource storageRef 'Microsoft.Storage/storageAccounts@2022-09-01' existing = { - name: storageAccountName -} - -resource keyVault_secrets 'Microsoft.KeyVault/vaults/secrets@2022-11-01' = { - name: storageRef.name - parent: keyVault - properties: { - attributes: { - enabled: true - exp: 1702648632 - nbf: 10000 + resource keyVault_accessPolicies 'accessPolicies@2023-07-01' = if (!empty(accessPolicies)) { + name: 'add' + properties: { + accessPolicies: formattedAccessPolicies } - value: storageRef.listKeys().keys[0].value + } + + resource keyVault_secrets 'secrets@2023-07-01' = { + name: storageRef.name + properties: storageSecretProperties } } @@ -105,10 +126,10 @@ resource keyVault_secrets 'Microsoft.KeyVault/vaults/secrets@2022-11-01' = { //============================================================================== @description('The resource ID of the key vault.') -output resourceId string = keyVault.id +output resourceId string = empty(existingKeyVaultName) ? keyVault.id : existingKeyVault.id @description('The name of the key vault.') -output name string = keyVault.name +output name string = empty(existingKeyVaultName) ? keyVault.name : existingKeyVault.name @description('The URI of the key vault.') -output uri string = keyVault.properties.vaultUri +output uri string = empty(existingKeyVaultName) ? keyVault.properties.vaultUri : existingKeyVault.properties.vaultUri