From 69055c1199e3d1dc920878ffae562bf33fa4c1c8 Mon Sep 17 00:00:00 2001 From: msbrett Date: Sun, 27 Oct 2024 20:03:36 -0700 Subject: [PATCH 01/19] vNet for private endpoint deployment --- src/templates/finops-hub/modules/vnet.bicep | 175 ++++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 src/templates/finops-hub/modules/vnet.bicep diff --git a/src/templates/finops-hub/modules/vnet.bicep b/src/templates/finops-hub/modules/vnet.bicep new file mode 100644 index 000000000..4c6888906 --- /dev/null +++ b/src/templates/finops-hub/modules/vnet.bicep @@ -0,0 +1,175 @@ + +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//============================================================================== +// Parameters +//============================================================================== + +@description('Required. Name of the hub. Used to ensure unique resource names.') +param hubName string + +@description('Address space for the workload. A /27 is required for the workload.') +param virtualNetworkAddressPrefix string = '10.20.30.0/27' + +@description('Optional. Azure location where all resources should be created. See https://aka.ms/azureregions. Default: (resource group location).') +param location string = resourceGroup().location + +//------------------------------------------------------------------------------ +// Variables +//------------------------------------------------------------------------------ + +var safeHubName = replace(replace(toLower(hubName), '-', ''), '_', '') +var vNetName = '${safeHubName}-vnet-${location}' +var nsgName = '${vNetName}-nsg' +var subnets = [ + { + name: 'finops-hub-subnet' + properties: { + addressPrefix: cidrSubnet(virtualNetworkAddressPrefix, 28, 0) + networkSecurityGroup: { + id: nsg.id + } + serviceEndpoints: [ + { + service: 'Microsoft.Storage' + } + ] + } + } + { + name: 'script-subnet' + properties: { + addressPrefix: cidrSubnet(virtualNetworkAddressPrefix, 28, 1) + networkSecurityGroup: { + id: nsg.id + } + delegations: [ + { + name: 'Microsoft.ContainerInstance/containerGroups' + properties: { + serviceName: 'Microsoft.ContainerInstance/containerGroups' + } + } + ] + serviceEndpoints: [ + { + service: 'Microsoft.Storage' + } + ] + } + } +] + +//------------------------------------------------------------------------------ +// Resources +//------------------------------------------------------------------------------ + +resource nsg 'Microsoft.Network/networkSecurityGroups@2023-11-01' = { + name: nsgName + location: location + properties: { + securityRules: [ + { + name: 'AllowVnetInBound' + properties: { + priority: 100 + direction: 'Inbound' + access: 'Allow' + protocol: '*' + sourcePortRange: '*' + destinationPortRange: '*' + sourceAddressPrefix: 'VirtualNetwork' + destinationAddressPrefix: 'VirtualNetwork' + } + } + { + name: 'AllowAzureLoadBalancerInBound' + properties: { + priority: 200 + direction: 'Inbound' + access: 'Allow' + protocol: '*' + sourcePortRange: '*' + destinationPortRange: '*' + sourceAddressPrefix: 'AzureLoadBalancer' + destinationAddressPrefix: '*' + } + } + { + name: 'DenyAllInBound' + properties: { + priority: 4096 + direction: 'Inbound' + access: 'Deny' + protocol: '*' + sourcePortRange: '*' + destinationPortRange: '*' + sourceAddressPrefix: '*' + destinationAddressPrefix: '*' + } + } + { + name: 'AllowVnetOutBound' + properties: { + priority: 100 + direction: 'Outbound' + access: 'Allow' + protocol: '*' + sourcePortRange: '*' + destinationPortRange: '*' + sourceAddressPrefix: 'VirtualNetwork' + destinationAddressPrefix: 'VirtualNetwork' + } + } + { + name: 'AllowInternetOutBound' + properties: { + priority: 200 + direction: 'Outbound' + access: 'Allow' + protocol: '*' + sourcePortRange: '*' + destinationPortRange: '*' + sourceAddressPrefix: '*' + destinationAddressPrefix: 'Internet' + } + } + { + name: 'DenyAllOutBound' + properties: { + priority: 4096 + direction: 'Outbound' + access: 'Deny' + protocol: '*' + sourcePortRange: '*' + destinationPortRange: '*' + sourceAddressPrefix: '*' + destinationAddressPrefix: '*' + } + } + ] + } +} + +resource vNet 'Microsoft.Network/virtualNetworks@2023-11-01' = { + name: vNetName + location: location + properties: { + addressSpace: { + addressPrefixes: [virtualNetworkAddressPrefix] + } + subnets: subnets + } +} + +//------------------------------------------------------------------------------ +// Outputs +//------------------------------------------------------------------------------ + +output vNetId string = vNet.id +output vNetName string = vNet.name +output vnetAddressSpace array = vNet.properties.addressSpace.addressPrefixes +output vNetSubnets array = vNet.properties.subnets +output finopsHubSubnetId string = vNet.properties.subnets[0].id +output scriptSubnetId string = vNet.properties.subnets[1].id From 02a12d8fbb0a3ae1584e06c89d52f9e4d08fd81a Mon Sep 17 00:00:00 2001 From: msbrett Date: Sun, 27 Oct 2024 20:03:55 -0700 Subject: [PATCH 02/19] Helper module for storage private endpoints --- .../finops-hub/modules/storageEndpoints.bicep | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 src/templates/finops-hub/modules/storageEndpoints.bicep diff --git a/src/templates/finops-hub/modules/storageEndpoints.bicep b/src/templates/finops-hub/modules/storageEndpoints.bicep new file mode 100644 index 000000000..42959a135 --- /dev/null +++ b/src/templates/finops-hub/modules/storageEndpoints.bicep @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//============================================================================== +// Parameters +//============================================================================== + +@description('Optional. Array of private endpoint connections. Pending ones will be approved.') +param privateEndpointConnections array = [] + +@description('Required. Name of the storage account.') +param storageAccountName string + +//============================================================================== +// Resources +//============================================================================== + +resource privateEndpointConnection 'Microsoft.Storage/storageAccounts/privateEndpointConnections@2023-04-01' = [ for privateEndpointConnection in privateEndpointConnections : if (privateEndpointConnection.properties.privateLinkServiceConnectionState.status == 'Pending') { + name: privateEndpointConnection.name + parent: storageAccount + properties: { + privateLinkServiceConnectionState: { + status: 'Approved' + description: 'Approved-by-pipeline' + actionRequired: 'None' + } + } +}] + +resource storageAccount 'Microsoft.Storage/storageAccounts@2023-04-01' existing = { + name: storageAccountName +} + +//============================================================================== +// Outputs +//============================================================================== + +output privateEndpointConnections array = storageAccount.properties.privateEndpointConnections From 8c0ebfff049d91c0f11bb6d8c750e70bc8556109 Mon Sep 17 00:00:00 2001 From: msbrett Date: Sun, 27 Oct 2024 20:04:12 -0700 Subject: [PATCH 03/19] Private endpoints for storage --- .../finops-hub/modules/storage.bicep | 214 +++++++++++++++++- 1 file changed, 211 insertions(+), 3 deletions(-) diff --git a/src/templates/finops-hub/modules/storage.bicep b/src/templates/finops-hub/modules/storage.bicep index 0e9a4ab8f..112affc7c 100644 --- a/src/templates/finops-hub/modules/storage.bicep +++ b/src/templates/finops-hub/modules/storage.bicep @@ -36,6 +36,18 @@ param msexportRetentionInDays int = 0 @description('Optional. Number of months of cost data to retain in the ingestion container. Default: 13.') param ingestionRetentionInMonths int = 13 +@description('Required. Id of the virtual network for private endpoints.') +param virtualNetworkId string + +@description('Required. Id of the subnet for private endpoints.') +param privateEndpointSubnetId string + +@description('Required. Id of the virtual network for running deployment scripts.') +param scriptSubnetId string + +@description('Optional. Enable public access to the data lake. Default: false.') +param enablePublicAccess bool + //------------------------------------------------------------------------------ // Variables //------------------------------------------------------------------------------ @@ -44,6 +56,7 @@ param ingestionRetentionInMonths int = 13 var safeHubName = replace(replace(toLower(hubName), '-', ''), '_', '') var storageAccountSuffix = uniqueSuffix var storageAccountName = '${take(safeHubName, 24 - length(storageAccountSuffix))}${storageAccountSuffix}' +var scriptStorageAccountName = '${take(safeHubName, 16 - length(storageAccountSuffix))}script${storageAccountSuffix}' var schemaFiles = { 'focuscost_1.0': loadTextContent('../schemas/focuscost_1.0.json') 'focuscost_1.0-preview(v1)': loadTextContent('../schemas/focuscost_1.0-preview(v1).json') @@ -60,6 +73,7 @@ var schemaFiles = { var blobUploadRbacRoles = [ 'ba92f5b4-2d11-453d-a403-e96b0029c9fe' // Storage Blob Data Contributor - https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#storage-blob-data-contributor 'e40ec5ca-96e0-45a2-b4ff-59039f2c2b59' // Managed Identity Contributor - https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#managed-identity-contributor + '69566ab7-960f-475b-8e7c-b3118f30c6bd' // Storage File Data Privileged Contributor - https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles/storage#storage-file-data-privileged-contributor ] //============================================================================== @@ -76,9 +90,181 @@ resource storageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' = { tags: union(tags, contains(tagsByResource, 'Microsoft.Storage/storageAccounts') ? tagsByResource['Microsoft.Storage/storageAccounts'] : {}) properties: { supportsHttpsTrafficOnly: true + allowSharedKeyAccess: true isHnsEnabled: true minimumTlsVersion: 'TLS1_2' allowBlobPublicAccess: false + publicNetworkAccess: 'Enabled' + networkAcls: { + bypass: 'AzureServices' + defaultAction: enablePublicAccess ? 'Allow' : 'Deny' + } + } +} + +resource scriptStorageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' = { + name: scriptStorageAccountName + location: location + sku: { + name: 'Standard_LRS' //sku + } + kind: 'StorageV2'// 'BlockBlobStorage' + tags: union(tags, contains(tagsByResource, 'Microsoft.Storage/storageAccounts') ? tagsByResource['Microsoft.Storage/storageAccounts'] : {}) + properties: { + supportsHttpsTrafficOnly: true + allowSharedKeyAccess: true + isHnsEnabled: false + minimumTlsVersion: 'TLS1_2' + allowBlobPublicAccess: false + publicNetworkAccess: 'Enabled' + networkAcls: { + bypass: 'AzureServices' + defaultAction: 'Deny' + virtualNetworkRules: [ + { + id: scriptSubnetId + action: 'Allow' + } + ] + } + } +} + +resource blobPrivateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' = { + name: 'privatelink.blob.${environment().suffixes.storage}' + location: 'global' + properties: {} +} + +resource dfsPrivateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' = { + name: 'privatelink.dfs.${environment().suffixes.storage}' + location: 'global' + properties: {} +} + +resource blobPrivateDnsZoneLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2024-06-01' = { + parent: blobPrivateDnsZone + name: '${replace(blobPrivateDnsZone.name, '.', '-')}-link' + location: 'global' + properties: { + registrationEnabled: false + virtualNetwork: { + id: virtualNetworkId + } + } +} + +resource dfsPrivateDnsZoneLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2024-06-01' = { + parent: dfsPrivateDnsZone + name: '${replace(dfsPrivateDnsZone.name, '.', '-')}-link' + location: 'global' + properties: { + registrationEnabled: false + virtualNetwork: { + id: virtualNetworkId + } + } +} + +resource blobEndpoint 'Microsoft.Network/privateEndpoints@2023-11-01' = { + name: '${storageAccount.name}-blob-ep' + location: location + properties: { + subnet: { + id: privateEndpointSubnetId + } + privateLinkServiceConnections: [ + { + name: 'blobLink' + properties: { + privateLinkServiceId: storageAccount.id + groupIds: ['blob'] + } + } + ] + } +} + +resource scriptEndpoint 'Microsoft.Network/privateEndpoints@2023-11-01' = { + name: '${scriptStorageAccount.name}-blob-ep' + location: location + properties: { + subnet: { + id: privateEndpointSubnetId + } + privateLinkServiceConnections: [ + { + name: 'scriptLink' + properties: { + privateLinkServiceId: scriptStorageAccount.id + groupIds: ['blob'] + } + } + ] + } +} + +resource dfsEndpoint 'Microsoft.Network/privateEndpoints@2023-11-01' = { + name: '${storageAccount.name}-dfs-ep' + location: location + properties: { + subnet: { + id: privateEndpointSubnetId + } + privateLinkServiceConnections: [ + { + name: 'dfsLink' + properties: { + privateLinkServiceId: storageAccount.id + groupIds: ['dfs'] + } + } + ] + } +} + +resource blobPrivateDnsZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2023-11-01' = { + name: 'blob-endpoint-zone' + parent: blobEndpoint + properties: { + privateDnsZoneConfigs: [ + { + name: blobPrivateDnsZone.name + properties: { + privateDnsZoneId: blobPrivateDnsZone.id + } + } + ] + } +} + +resource scriptPrivateDnsZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2023-11-01' = { + name: 'blob-endpoint-zone' + parent: scriptEndpoint + properties: { + privateDnsZoneConfigs: [ + { + name: blobPrivateDnsZone.name + properties: { + privateDnsZoneId: blobPrivateDnsZone.id + } + } + ] + } +} + +resource dfsPrivateDnsZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2023-11-01' = { + name: 'dfs-endpoint-zone' + parent: dfsEndpoint + properties: { + privateDnsZoneConfigs: [ + { + name: dfsPrivateDnsZone.name + properties: { + privateDnsZoneId: dfsPrivateDnsZone.id + } + } + ] } } @@ -140,8 +326,8 @@ resource identityRoleAssignments 'Microsoft.Authorization/roleAssignments@2022-0 } }] -resource uploadSettings 'Microsoft.Resources/deploymentScripts@2020-10-01' = { - name: '${storageAccountName}_uploadSettings' +resource uploadSettings 'Microsoft.Resources/deploymentScripts@2023-08-01' = { + name: '${safeHubName}_uploadSettings' kind: 'AzurePowerShell' // chinaeast2 is the only region in China that supports deployment scripts location: startsWith(location, 'china') ? 'chinaeast2' : location @@ -155,9 +341,13 @@ resource uploadSettings 'Microsoft.Resources/deploymentScripts@2020-10-01' = { dependsOn: [ configContainer identityRoleAssignments + blobEndpoint + blobPrivateDnsZoneGroup + scriptEndpoint + scriptPrivateDnsZoneGroup ] properties: { - azPowerShellVersion: '8.0' + azPowerShellVersion: '9.0' retentionInterval: 'PT1H' environmentVariables: [ { @@ -190,6 +380,18 @@ resource uploadSettings 'Microsoft.Resources/deploymentScripts@2020-10-01' = { } ] scriptContent: loadTextContent('./scripts/Copy-FileToAzureBlob.ps1') + storageAccountSettings: { + storageAccountName: scriptStorageAccount.name + //storageAccountKey: storageAccount.listKeys().keys[0].value + } + containerSettings: { + containerGroupName: '${scriptStorageAccount.name}cg' + subnetIds: [ + { + id: scriptSubnetId + } + ] + } } } @@ -203,6 +405,12 @@ output resourceId string = storageAccount.id @description('The name of the storage account.') output name string = storageAccount.name +@description('The resource ID of the storage account.') +output scriptStorageAccountResourceId string = scriptStorageAccount.id + +@description('The name of the storage account.') +output scriptStorageAccountName string = scriptStorageAccount.name + @description('The name of the container used for configuration settings.') output configContainer string = configContainer.name From 53374554663c0daf0fd8dbc3b3af41872693d4f7 Mon Sep 17 00:00:00 2001 From: msbrett Date: Sun, 27 Oct 2024 20:04:24 -0700 Subject: [PATCH 04/19] Private endpoints for KeyVault --- .../finops-hub/modules/keyVault.bicep | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/src/templates/finops-hub/modules/keyVault.bicep b/src/templates/finops-hub/modules/keyVault.bicep index e082cf861..4fd16aa07 100644 --- a/src/templates/finops-hub/modules/keyVault.bicep +++ b/src/templates/finops-hub/modules/keyVault.bicep @@ -34,6 +34,12 @@ param tags object = {} @description('Optional. Tags to apply to resources based on their resource type. Resource type specific tags will be merged with tags for all resources.') param tagsByResource object = {} +@description('Required. Id of the virtual network for private endpoints.') +param virtualNetworkId string + +@description('Required. Id of the subnet for private endpoints.') +param privateEndpointSubnetId string + //------------------------------------------------------------------------------ // Variables //------------------------------------------------------------------------------ @@ -43,6 +49,7 @@ var keyVaultPrefix = '${replace(hubName, '_', '-')}-vault' var keyVaultSuffix = '-${uniqueSuffix}' var keyVaultName = replace('${take(keyVaultPrefix, 24 - length(keyVaultSuffix))}${keyVaultSuffix}', '--', '-') var keyVaultSecretName = '${toLower(hubName)}-storage-key' +var keyVaultPrivateDnsZoneName = 'privatelink${replace(environment().suffixes.keyvaultDns, 'vault', 'vaultcore')}' var formattedAccessPolicies = [for accessPolicy in accessPolicies: { applicationId: contains(accessPolicy, 'applicationId') ? accessPolicy.applicationId : '' @@ -74,6 +81,10 @@ resource keyVault 'Microsoft.KeyVault/vaults@2023-02-01' = { name: startsWith(location, 'china') ? 'standard' : sku family: 'A' } + networkAcls: { + bypass: 'AzureServices' + defaultAction: 'Deny' + } } } @@ -98,6 +109,59 @@ resource keyVault_secret 'Microsoft.KeyVault/vaults/secrets@2023-02-01' = if (!e } } +resource keyVaultPrivateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' = { + name: keyVaultPrivateDnsZoneName + location: 'global' + properties: {} +} + +resource keyVaultPrivateDnsZoneLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2024-06-01' = { + name: '${replace(keyVaultPrivateDnsZone.name, '.', '-')}-link' + location: 'global' + parent: keyVaultPrivateDnsZone + properties: { + virtualNetwork: { + id: virtualNetworkId + } + registrationEnabled: false + } +} + +resource keyVaultEndpoint 'Microsoft.Network/privateEndpoints@2023-11-01' = { + name: '${keyVault.name}-ep' + location: location + properties: { + subnet: { + id: privateEndpointSubnetId + } + privateLinkServiceConnections: [ + { + name: 'keyVaultLink' + properties: { + privateLinkServiceId: keyVault.id + groupIds: ['vault'] + } + } + ] + } +} + +resource keyVaultPrivateDnsZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2023-11-01' = { + name: 'keyvault-endpoint-zone' + parent: keyVaultEndpoint + properties: { + privateDnsZoneConfigs: [ + { + name: keyVaultPrivateDnsZone.name + properties: { + privateDnsZoneId: keyVaultPrivateDnsZone.id + } + } + ] + } +} + + //============================================================================== // Outputs //============================================================================== From a404f43df76c623c3ddd5d39cf66700ea849afe6 Mon Sep 17 00:00:00 2001 From: msbrett Date: Sun, 27 Oct 2024 20:05:02 -0700 Subject: [PATCH 05/19] Managed private network and endpoint --- .../finops-hub/modules/dataFactory.bicep | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/src/templates/finops-hub/modules/dataFactory.bicep b/src/templates/finops-hub/modules/dataFactory.bicep index 48985e1e1..a2d213c29 100644 --- a/src/templates/finops-hub/modules/dataFactory.bicep +++ b/src/templates/finops-hub/modules/dataFactory.bicep @@ -66,6 +66,8 @@ var datasetPropsDefault = { var safeExportContainerName = replace('${exportContainerName}', '-', '_') var safeIngestionContainerName = replace('${ingestionContainerName}', '-', '_') var safeConfigContainerName = replace('${configContainerName}', '-', '_') +var managedVnetName = 'default' +var managedIntegrationRuntimeName = 'AutoResolveIntegrationRuntime' // All hub triggers (used to auto-start) var fileAddedExportTriggerName = '${safeExportContainerName}_FileAdded' @@ -120,6 +122,69 @@ module azuretimezones 'azuretimezones.bicep' = { } } +resource managedVirtualNetwork 'Microsoft.DataFactory/factories/managedVirtualNetworks@2018-06-01' = { + name: managedVnetName + parent: dataFactory + properties: {} +} + +resource managedIntegrationRuntime 'Microsoft.DataFactory/factories/integrationRuntimes@2018-06-01' = { + name: managedIntegrationRuntimeName + parent: dataFactory + properties: { + type: 'Managed' + managedVirtualNetwork: { + referenceName: managedVnetName + type: 'ManagedVirtualNetworkReference' + } + typeProperties: { + computeProperties: { + location: 'AutoResolve' + } + } + } + dependsOn: [ + managedVirtualNetwork + ] +} + +resource storageManagedPrivateEndpoint 'Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints@2018-06-01' = { + name: storageAccount.name + parent: managedVirtualNetwork + dependsOn: [ + identityRoleAssignments + ] + properties: { + name: storageAccount.name + groupId: 'dfs' + privateLinkResourceId: storageAccount.id + fqdns: [ + storageAccount.properties.primaryEndpoints.dfs + ] + } +} + +module getPrivateEndpointConnections 'storageEndpoints.bicep' = { + name: 'GetPrivateEndpointConnections' + dependsOn: [ + storageManagedPrivateEndpoint + ] + params: { + storageAccountName: storageAccount.name + } +} + +module approvePrivateEndpointConnections 'storageEndpoints.bicep' = { + name: 'ApprovePrivateEndpointConnections' + dependsOn: [ + getPrivateEndpointConnections + ] + params: { + storageAccountName: storageAccount.name + privateEndpointConnections: getPrivateEndpointConnections.outputs.privateEndpointConnections + } +} + //------------------------------------------------------------------------------ // Identities and RBAC //------------------------------------------------------------------------------ @@ -265,6 +330,10 @@ resource linkedService_storageAccount 'Microsoft.DataFactory/factories/linkedser typeProperties: { url: reference('Microsoft.Storage/storageAccounts/${storageAccount.name}', '2021-08-01').primaryEndpoints.dfs } + connectVia: { + referenceName: managedIntegrationRuntime.name + type: 'IntegrationRuntimeReference' + } } } From f0ec1c9f9c1d02c08fe1409e6c523d42365a4dec Mon Sep 17 00:00:00 2001 From: msbrett Date: Sun, 27 Oct 2024 20:05:44 -0700 Subject: [PATCH 06/19] Private endpoint support --- src/templates/finops-hub/main.bicep | 8 +++++++ src/templates/finops-hub/modules/hub.bicep | 25 ++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/src/templates/finops-hub/main.bicep b/src/templates/finops-hub/main.bicep index cc7d08b15..867f84f3b 100644 --- a/src/templates/finops-hub/main.bicep +++ b/src/templates/finops-hub/main.bicep @@ -45,6 +45,12 @@ param remoteHubStorageUri string = '' @secure() param remoteHubStorageKey string = '' +@description('Optional. Enable public access to the data lake. Default: false.') +param enablePublicAccess bool = true + +@description('Address space for the workload. A /27 is required for the workload.') +param virtualNetworkAddressPrefix string = '10.20.30.0/27' + //============================================================================== // Resources //============================================================================== @@ -63,6 +69,8 @@ module hub 'modules/hub.bicep' = { ingestionRetentionInMonths: ingestionRetentionInMonths remoteHubStorageUri: remoteHubStorageUri remoteHubStorageKey: remoteHubStorageKey + enablePublicAccess: enablePublicAccess + virtualNetworkAddressPrefix: virtualNetworkAddressPrefix } } diff --git a/src/templates/finops-hub/modules/hub.bicep b/src/templates/finops-hub/modules/hub.bicep index 0f7ebe6ed..9a0f2e754 100644 --- a/src/templates/finops-hub/modules/hub.bicep +++ b/src/templates/finops-hub/modules/hub.bicep @@ -43,6 +43,12 @@ param remoteHubStorageUri string = '' @secure() param remoteHubStorageKey string = '' +@description('Address space for the workload. A /27 is required for the workload.') +param virtualNetworkAddressPrefix string = '10.20.30.0/27' + +@description('Optional. Enable public access to the data lake. Default: false.') +param enablePublicAccess bool = true + @description('Optional. Enable telemetry to track anonymous module usage trends, monitor for bugs, and improve future releases.') param enableDefaultTelemetry bool = true @@ -121,6 +127,19 @@ resource defaultTelemetry 'Microsoft.Resources/deployments@2022-09-01' = if (ena } } +//------------------------------------------------------------------------------ +// Virtual network +//------------------------------------------------------------------------------ + +module vnet 'vnet.bicep' = { + name: 'vnet' + params: { + hubName: hubName + location: location + virtualNetworkAddressPrefix: virtualNetworkAddressPrefix + } +} + //------------------------------------------------------------------------------ // ADLSv2 storage account for staging and archive //------------------------------------------------------------------------------ @@ -137,6 +156,10 @@ module storage 'storage.bicep' = { scopesToMonitor: scopesToMonitor msexportRetentionInDays: exportRetentionInDays ingestionRetentionInMonths: ingestionRetentionInMonths + virtualNetworkId: vnet.outputs.vNetId + privateEndpointSubnetId: vnet.outputs.finopsHubSubnetId + scriptSubnetId: vnet.outputs.scriptSubnetId + enablePublicAccess: enablePublicAccess } } @@ -190,6 +213,8 @@ module keyVault 'keyVault.bicep' = { tags: resourceTags tagsByResource: tagsByResource storageAccountKey: remoteHubStorageKey + virtualNetworkId: vnet.outputs.vNetId + privateEndpointSubnetId: vnet.outputs.finopsHubSubnetId accessPolicies: [ { objectId: dataFactory.identity.principalId From b39c629e2414857e2971b2493f21506291846011 Mon Sep 17 00:00:00 2001 From: msbrett Date: Sun, 27 Oct 2024 20:17:02 -0700 Subject: [PATCH 07/19] changelog --- docs/_resources/changelog.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/_resources/changelog.md b/docs/_resources/changelog.md index 1a379a139..68ed8fda0 100644 --- a/docs/_resources/changelog.md +++ b/docs/_resources/changelog.md @@ -65,6 +65,12 @@ Legend: > 2. Auto-backfill – Backfill historical data from Microsoft Cost Management. > 3. Retention – Configure how long you want to keep Cost Management exports and normalized data in storage. > 4. ETL pipelile – Add support for parquet files created by Cost Management exports. +> 5. Private endpoints support. +> - Added private endpoints for storage account & Keyvault. +> - Added managed virtual network & storage endpoint for Azure Data Factory Runtime. +> - All data processing now happens within a vNet. +> - Added param to disable external access to data lake +> - Added param to specify subnet range of vnet - minumum size = /27 📊 Power BI reports {: .fs-5 .fw-500 .mt-4 mb-0 } From cefe1c72d6f9ca86940417e986f03f68cc9994e7 Mon Sep 17 00:00:00 2001 From: msbrett Date: Thu, 31 Oct 2024 08:43:05 -0700 Subject: [PATCH 08/19] typo --- src/templates/finops-hub/modules/vnet.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/templates/finops-hub/modules/vnet.bicep b/src/templates/finops-hub/modules/vnet.bicep index 4c6888906..389bf6278 100644 --- a/src/templates/finops-hub/modules/vnet.bicep +++ b/src/templates/finops-hub/modules/vnet.bicep @@ -169,7 +169,7 @@ resource vNet 'Microsoft.Network/virtualNetworks@2023-11-01' = { output vNetId string = vNet.id output vNetName string = vNet.name -output vnetAddressSpace array = vNet.properties.addressSpace.addressPrefixes +output vNetAddressSpace array = vNet.properties.addressSpace.addressPrefixes output vNetSubnets array = vNet.properties.subnets output finopsHubSubnetId string = vNet.properties.subnets[0].id output scriptSubnetId string = vNet.properties.subnets[1].id From 2ea12f9f5672e93c37b9945310042d9e9064afb7 Mon Sep 17 00:00:00 2001 From: msbrett Date: Thu, 31 Oct 2024 08:43:32 -0700 Subject: [PATCH 09/19] Move dependency to top --- src/templates/finops-hub/modules/storageEndpoints.bicep | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/templates/finops-hub/modules/storageEndpoints.bicep b/src/templates/finops-hub/modules/storageEndpoints.bicep index 42959a135..ad3916f77 100644 --- a/src/templates/finops-hub/modules/storageEndpoints.bicep +++ b/src/templates/finops-hub/modules/storageEndpoints.bicep @@ -15,6 +15,10 @@ param storageAccountName string // Resources //============================================================================== +resource storageAccount 'Microsoft.Storage/storageAccounts@2023-04-01' existing = { + name: storageAccountName +} + resource privateEndpointConnection 'Microsoft.Storage/storageAccounts/privateEndpointConnections@2023-04-01' = [ for privateEndpointConnection in privateEndpointConnections : if (privateEndpointConnection.properties.privateLinkServiceConnectionState.status == 'Pending') { name: privateEndpointConnection.name parent: storageAccount @@ -27,10 +31,6 @@ resource privateEndpointConnection 'Microsoft.Storage/storageAccounts/privateEnd } }] -resource storageAccount 'Microsoft.Storage/storageAccounts@2023-04-01' existing = { - name: storageAccountName -} - //============================================================================== // Outputs //============================================================================== From 291ae4362cf19b8da8473fb6a451c6f6cd6f6d2f Mon Sep 17 00:00:00 2001 From: msbrett Date: Thu, 31 Oct 2024 08:43:55 -0700 Subject: [PATCH 10/19] Fix script name and doc perms --- src/templates/finops-hub/modules/storage.bicep | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/templates/finops-hub/modules/storage.bicep b/src/templates/finops-hub/modules/storage.bicep index 112affc7c..9a36ed0c2 100644 --- a/src/templates/finops-hub/modules/storage.bicep +++ b/src/templates/finops-hub/modules/storage.bicep @@ -70,6 +70,9 @@ var schemaFiles = { } // Roles needed to auto-start triggers +// Storage Blob Data Contributor - used by deployment scripts to write data to blob storage +// Storage File Data Privileged Contributor - used by deployment scripts to write data to blob storage +// https://learn.microsoft.com/en-us/azure/azure-resource-manager/templates/deployment-script-template#use-existing-storage-account var blobUploadRbacRoles = [ 'ba92f5b4-2d11-453d-a403-e96b0029c9fe' // Storage Blob Data Contributor - https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#storage-blob-data-contributor 'e40ec5ca-96e0-45a2-b4ff-59039f2c2b59' // Managed Identity Contributor - https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#managed-identity-contributor @@ -327,7 +330,7 @@ resource identityRoleAssignments 'Microsoft.Authorization/roleAssignments@2022-0 }] resource uploadSettings 'Microsoft.Resources/deploymentScripts@2023-08-01' = { - name: '${safeHubName}_uploadSettings' + name: '${storageAccountName}_uploadSettings' kind: 'AzurePowerShell' // chinaeast2 is the only region in China that supports deployment scripts location: startsWith(location, 'china') ? 'chinaeast2' : location From 5282f8f2c39f2954f0fc648eb8782dd3fa881789 Mon Sep 17 00:00:00 2001 From: msbrett Date: Thu, 31 Oct 2024 08:44:23 -0700 Subject: [PATCH 11/19] Typo --- src/templates/finops-hub/main.bicep | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/templates/finops-hub/main.bicep b/src/templates/finops-hub/main.bicep index 867f84f3b..c7d131d79 100644 --- a/src/templates/finops-hub/main.bicep +++ b/src/templates/finops-hub/main.bicep @@ -45,10 +45,10 @@ param remoteHubStorageUri string = '' @secure() param remoteHubStorageKey string = '' -@description('Optional. Enable public access to the data lake. Default: false.') +@description('Optional. Enable public access to the data lake. Default: true.') param enablePublicAccess bool = true -@description('Address space for the workload. A /27 is required for the workload.') +@description('Optional. Address space for the workload. A /27 is required for the workload. Default: "10.20.30.0/27".') param virtualNetworkAddressPrefix string = '10.20.30.0/27' //============================================================================== From 35f87f40f0fd587fa913d87c90365ddd295a1dc8 Mon Sep 17 00:00:00 2001 From: msbrett Date: Thu, 31 Oct 2024 08:45:29 -0700 Subject: [PATCH 12/19] Managed PE for KeyVault --- .../modules/keyVaultEndpoints.bicep | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 src/templates/finops-hub/modules/keyVaultEndpoints.bicep diff --git a/src/templates/finops-hub/modules/keyVaultEndpoints.bicep b/src/templates/finops-hub/modules/keyVaultEndpoints.bicep new file mode 100644 index 000000000..5de6e0e86 --- /dev/null +++ b/src/templates/finops-hub/modules/keyVaultEndpoints.bicep @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//============================================================================== +// Parameters +//============================================================================== + +@description('Optional. Array of private endpoint connections. Pending ones will be approved.') +param privateEndpointConnections array = [] + +@description('Required. Name of the KeyVault.') +param keyVaultName string + +//============================================================================== +// Resources +//============================================================================== + +resource keyVault 'Microsoft.KeyVault/vaults@2023-07-01' existing = { + name: keyVaultName +} + +resource privateEndpointConnection 'Microsoft.KeyVault/vaults/privateEndpointConnections@2023-07-01' = [ for privateEndpointConnection in privateEndpointConnections : if (privateEndpointConnection.properties.privateLinkServiceConnectionState.status == 'Pending') { + name: privateEndpointConnection.name + parent: keyVault + properties: { + privateLinkServiceConnectionState: { + status: 'Approved' + description: 'Approved-by-pipeline' + } + } +}] + +//============================================================================== +// Outputs +//============================================================================== + +output privateEndpointConnections array = keyVault.properties.privateEndpointConnections From fdfb0c2eb7a706ac0bd09accf7a79df95ff6c09f Mon Sep 17 00:00:00 2001 From: msbrett Date: Thu, 31 Oct 2024 08:45:47 -0700 Subject: [PATCH 13/19] Add managed PE for KeyVault --- .../finops-hub/modules/dataFactory.bicep | 45 +++++++++++++++++-- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/src/templates/finops-hub/modules/dataFactory.bicep b/src/templates/finops-hub/modules/dataFactory.bicep index a2d213c29..a064cee09 100644 --- a/src/templates/finops-hub/modules/dataFactory.bicep +++ b/src/templates/finops-hub/modules/dataFactory.bicep @@ -164,7 +164,7 @@ resource storageManagedPrivateEndpoint 'Microsoft.DataFactory/factories/managedV } } -module getPrivateEndpointConnections 'storageEndpoints.bicep' = { +module getStoragePrivateEndpointConnections 'storageEndpoints.bicep' = { name: 'GetPrivateEndpointConnections' dependsOn: [ storageManagedPrivateEndpoint @@ -174,14 +174,51 @@ module getPrivateEndpointConnections 'storageEndpoints.bicep' = { } } -module approvePrivateEndpointConnections 'storageEndpoints.bicep' = { +module approveStoragePrivateEndpointConnections 'storageEndpoints.bicep' = { name: 'ApprovePrivateEndpointConnections' dependsOn: [ - getPrivateEndpointConnections + getStoragePrivateEndpointConnections ] params: { storageAccountName: storageAccount.name - privateEndpointConnections: getPrivateEndpointConnections.outputs.privateEndpointConnections + privateEndpointConnections: getStoragePrivateEndpointConnections.outputs.privateEndpointConnections + } +} + +resource keyVaultManagedPrivateEndpoint 'Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints@2018-06-01' = { + name: keyVault.name + parent: managedVirtualNetwork + dependsOn: [ + identityRoleAssignments + ] + properties: { + name: keyVault.name + groupId: 'vault' + privateLinkResourceId: keyVault.id + fqdns: [ + keyVault.properties.vaultUri + ] + } +} + +module getKeyVaultPrivateEndpointConnections 'keyVaultEndpoints.bicep' = { + name: 'GetKeyVaultPrivateEndpointConnections' + dependsOn: [ + keyVaultManagedPrivateEndpoint + ] + params: { + keyVaultName: keyVault.name + } +} + +module approveKeyVaultPrivateEndpointConnections 'keyVaultEndpoints.bicep' = { + name: 'ApproveKeyVaultPrivateEndpointConnections' + dependsOn: [ + getKeyVaultPrivateEndpointConnections + ] + params: { + keyVaultName: keyVault.name + privateEndpointConnections: getKeyVaultPrivateEndpointConnections.outputs.privateEndpointConnections } } From ae4c00920b9cb0be23132cd97a8a0dfc5707f129 Mon Sep 17 00:00:00 2001 From: msbrett Date: Thu, 31 Oct 2024 08:45:58 -0700 Subject: [PATCH 14/19] Typo --- src/templates/finops-hub/modules/hub.bicep | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/templates/finops-hub/modules/hub.bicep b/src/templates/finops-hub/modules/hub.bicep index 9a0f2e754..680487db1 100644 --- a/src/templates/finops-hub/modules/hub.bicep +++ b/src/templates/finops-hub/modules/hub.bicep @@ -43,10 +43,10 @@ param remoteHubStorageUri string = '' @secure() param remoteHubStorageKey string = '' -@description('Address space for the workload. A /27 is required for the workload.') +@description('Optional. Address space for the workload. A /27 is required for the workload. Default: "10.20.30.0/27".') param virtualNetworkAddressPrefix string = '10.20.30.0/27' -@description('Optional. Enable public access to the data lake. Default: false.') +@description('Optional. Enable public access to the data lake. Default: true.') param enablePublicAccess bool = true @description('Optional. Enable telemetry to track anonymous module usage trends, monitor for bugs, and improve future releases.') From abc6406371961c4d8dfeeb45c052c4123819a157 Mon Sep 17 00:00:00 2001 From: msbrett Date: Thu, 31 Oct 2024 09:38:59 -0700 Subject: [PATCH 15/19] Typo --- src/templates/finops-hub/modules/dataFactory.bicep | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/templates/finops-hub/modules/dataFactory.bicep b/src/templates/finops-hub/modules/dataFactory.bicep index a064cee09..a7281019d 100644 --- a/src/templates/finops-hub/modules/dataFactory.bicep +++ b/src/templates/finops-hub/modules/dataFactory.bicep @@ -165,7 +165,7 @@ resource storageManagedPrivateEndpoint 'Microsoft.DataFactory/factories/managedV } module getStoragePrivateEndpointConnections 'storageEndpoints.bicep' = { - name: 'GetPrivateEndpointConnections' + name: 'GetStoragePrivateEndpointConnections' dependsOn: [ storageManagedPrivateEndpoint ] @@ -175,7 +175,7 @@ module getStoragePrivateEndpointConnections 'storageEndpoints.bicep' = { } module approveStoragePrivateEndpointConnections 'storageEndpoints.bicep' = { - name: 'ApprovePrivateEndpointConnections' + name: 'ApproveStoragePrivateEndpointConnections' dependsOn: [ getStoragePrivateEndpointConnections ] From 5652e1ba5711f29831187b4486e7876822c65ea8 Mon Sep 17 00:00:00 2001 From: msbrett Date: Thu, 31 Oct 2024 09:39:22 -0700 Subject: [PATCH 16/19] Derive name from ID field. --- src/templates/finops-hub/modules/keyVaultEndpoints.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/templates/finops-hub/modules/keyVaultEndpoints.bicep b/src/templates/finops-hub/modules/keyVaultEndpoints.bicep index 5de6e0e86..4dab785e2 100644 --- a/src/templates/finops-hub/modules/keyVaultEndpoints.bicep +++ b/src/templates/finops-hub/modules/keyVaultEndpoints.bicep @@ -20,7 +20,7 @@ resource keyVault 'Microsoft.KeyVault/vaults@2023-07-01' existing = { } resource privateEndpointConnection 'Microsoft.KeyVault/vaults/privateEndpointConnections@2023-07-01' = [ for privateEndpointConnection in privateEndpointConnections : if (privateEndpointConnection.properties.privateLinkServiceConnectionState.status == 'Pending') { - name: privateEndpointConnection.name + name: last(array(split(privateEndpointConnection.id, '/'))) parent: keyVault properties: { privateLinkServiceConnectionState: { From 96a95ca29dd0c441b560566681142d90e346e91e Mon Sep 17 00:00:00 2001 From: msbrett Date: Thu, 31 Oct 2024 09:39:29 -0700 Subject: [PATCH 17/19] Derive name from ID field. --- src/templates/finops-hub/modules/storageEndpoints.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/templates/finops-hub/modules/storageEndpoints.bicep b/src/templates/finops-hub/modules/storageEndpoints.bicep index ad3916f77..60ef5d052 100644 --- a/src/templates/finops-hub/modules/storageEndpoints.bicep +++ b/src/templates/finops-hub/modules/storageEndpoints.bicep @@ -20,7 +20,7 @@ resource storageAccount 'Microsoft.Storage/storageAccounts@2023-04-01' existing } resource privateEndpointConnection 'Microsoft.Storage/storageAccounts/privateEndpointConnections@2023-04-01' = [ for privateEndpointConnection in privateEndpointConnections : if (privateEndpointConnection.properties.privateLinkServiceConnectionState.status == 'Pending') { - name: privateEndpointConnection.name + name: last(array(split(privateEndpointConnection.id, '/'))) parent: storageAccount properties: { privateLinkServiceConnectionState: { From 5105124b2996541e6878a48820c9cdb9e5d9c9e9 Mon Sep 17 00:00:00 2001 From: msbrett Date: Thu, 31 Oct 2024 10:05:45 -0700 Subject: [PATCH 18/19] template.md --- docs/_reporting/hubs/template.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/_reporting/hubs/template.md b/docs/_reporting/hubs/template.md index 42c15f9e0..b26b1364c 100644 --- a/docs/_reporting/hubs/template.md +++ b/docs/_reporting/hubs/template.md @@ -94,6 +94,8 @@ Please ensure the following prerequisites are met before deploying this template | **ingestionRetentionInMonths** | int | Optional. Number of months of cost data to retain in the ingestion container. | 13 | | **remoteHubStorageUri** | string | Optional. Storage account to push data to for ingestion into a remote hub. | | | **remoteHubStorageKey** | string | Optional. Storage account key to use when pushing data to a remote hub. | | +| **enablePublicAccess** | string | Optional. Disable public access to the datalake (storage firewall). | False | +| **virtualNetworkAddressPrefix** | string | Optional. IP Address range for the private vNet used by the toolkit. /27 recommended to avoid wasting IP's as the deployment will split it into 2 x /28 subnets. | '10.20.30.0/27' |
@@ -114,6 +116,7 @@ Resources use the following naming convention: `--script` storage account (Data Lake Storage Gen2) for deployment scripts. - `-engine-` Data Factory instance - Pipelines: - `msexports_ExecuteETL` – Queues the `msexports_ETL_ingestion` pipeline to account for Data Factory pipeline trigger limits. @@ -130,6 +133,9 @@ Resources use the following naming convention: `--store` - Managed private endpoint for storage account. + - `-vault-` - Managed private endpoint for Azure Key Vault. - `-vault-` Key Vault instance - Secrets: - Data Factory system managed identity From a70e0555b139b04c0eafda9fc862000a11fb231d Mon Sep 17 00:00:00 2001 From: msbrett Date: Thu, 31 Oct 2024 10:07:45 -0700 Subject: [PATCH 19/19] Formatting --- docs/_reporting/hubs/template.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/_reporting/hubs/template.md b/docs/_reporting/hubs/template.md index b26b1364c..5a7f9e377 100644 --- a/docs/_reporting/hubs/template.md +++ b/docs/_reporting/hubs/template.md @@ -94,8 +94,8 @@ Please ensure the following prerequisites are met before deploying this template | **ingestionRetentionInMonths** | int | Optional. Number of months of cost data to retain in the ingestion container. | 13 | | **remoteHubStorageUri** | string | Optional. Storage account to push data to for ingestion into a remote hub. | | | **remoteHubStorageKey** | string | Optional. Storage account key to use when pushing data to a remote hub. | | -| **enablePublicAccess** | string | Optional. Disable public access to the datalake (storage firewall). | False | -| **virtualNetworkAddressPrefix** | string | Optional. IP Address range for the private vNet used by the toolkit. /27 recommended to avoid wasting IP's as the deployment will split it into 2 x /28 subnets. | '10.20.30.0/27' | +| **enablePublicAccess** | string | Optional. Disable public access to the datalake (storage firewall). | False | +| **virtualNetworkAddressPrefix**| string | Optional. IP Address range for the private vNet used by the toolkit. /27 recommended to avoid wasting IP's as the deployment will split it into 2 x /28 subnets. | '10.20.30.0/27' |