From 21586fe4237befd3f0083eb355626fb652a15cd7 Mon Sep 17 00:00:00 2001 From: Lyle Xu Date: Mon, 15 Apr 2024 21:15:21 +0800 Subject: [PATCH] add aks-store-demo --- .../AKS-Store-Demo/abbreviations.json | 136 ++++++++ .../app/aks-managed-cluster.bicep | 136 ++++++++ Environments/AKS-Store-Demo/app/db.bicep | 55 +++ .../AKS-Store-Demo/app/get-keys.bicep | 35 ++ .../AKS-Store-Demo/app/identity.bicep | 53 +++ Environments/AKS-Store-Demo/app/monitor.bicep | 12 + .../AKS-Store-Demo/app/observability.bicep | 327 ++++++++++++++++++ .../AKS-Store-Demo/app/servicebus.bicep | 70 ++++ .../core/ai/cognitiveservices.bicep | 53 +++ .../core/database/cosmos/cosmos-account.bicep | 49 +++ .../cosmos/mongo/cosmos-mongo-account.bicep | 23 ++ .../cosmos/mongo/cosmos-mongo-db.bicep | 47 +++ .../cosmos/sql/cosmos-sql-account.bicep | 22 ++ .../database/cosmos/sql/cosmos-sql-db.bicep | 74 ++++ .../cosmos/sql/cosmos-sql-role-assign.bicep | 19 + .../cosmos/sql/cosmos-sql-role-def.bicep | 30 ++ .../core/host/container-registry.bicep | 83 +++++ .../core/monitor/loganalytics.bicep | 22 ++ .../core/security/keyvault-access.bicep | 22 ++ .../core/security/keyvault-secret.bicep | 31 ++ .../core/security/keyvault.bicep | 26 ++ .../core/security/registry-access.bicep | 19 + .../AKS-Store-Demo/core/security/role.bicep | 21 ++ Environments/AKS-Store-Demo/main.bicep | 227 ++++++++++++ Environments/AKS-Store-Demo/manifest.yaml | 28 ++ 25 files changed, 1620 insertions(+) create mode 100644 Environments/AKS-Store-Demo/abbreviations.json create mode 100644 Environments/AKS-Store-Demo/app/aks-managed-cluster.bicep create mode 100644 Environments/AKS-Store-Demo/app/db.bicep create mode 100644 Environments/AKS-Store-Demo/app/get-keys.bicep create mode 100644 Environments/AKS-Store-Demo/app/identity.bicep create mode 100644 Environments/AKS-Store-Demo/app/monitor.bicep create mode 100644 Environments/AKS-Store-Demo/app/observability.bicep create mode 100644 Environments/AKS-Store-Demo/app/servicebus.bicep create mode 100644 Environments/AKS-Store-Demo/core/ai/cognitiveservices.bicep create mode 100644 Environments/AKS-Store-Demo/core/database/cosmos/cosmos-account.bicep create mode 100644 Environments/AKS-Store-Demo/core/database/cosmos/mongo/cosmos-mongo-account.bicep create mode 100644 Environments/AKS-Store-Demo/core/database/cosmos/mongo/cosmos-mongo-db.bicep create mode 100644 Environments/AKS-Store-Demo/core/database/cosmos/sql/cosmos-sql-account.bicep create mode 100644 Environments/AKS-Store-Demo/core/database/cosmos/sql/cosmos-sql-db.bicep create mode 100644 Environments/AKS-Store-Demo/core/database/cosmos/sql/cosmos-sql-role-assign.bicep create mode 100644 Environments/AKS-Store-Demo/core/database/cosmos/sql/cosmos-sql-role-def.bicep create mode 100644 Environments/AKS-Store-Demo/core/host/container-registry.bicep create mode 100644 Environments/AKS-Store-Demo/core/monitor/loganalytics.bicep create mode 100644 Environments/AKS-Store-Demo/core/security/keyvault-access.bicep create mode 100644 Environments/AKS-Store-Demo/core/security/keyvault-secret.bicep create mode 100644 Environments/AKS-Store-Demo/core/security/keyvault.bicep create mode 100644 Environments/AKS-Store-Demo/core/security/registry-access.bicep create mode 100644 Environments/AKS-Store-Demo/core/security/role.bicep create mode 100644 Environments/AKS-Store-Demo/main.bicep create mode 100644 Environments/AKS-Store-Demo/manifest.yaml diff --git a/Environments/AKS-Store-Demo/abbreviations.json b/Environments/AKS-Store-Demo/abbreviations.json new file mode 100644 index 00000000..289a08aa --- /dev/null +++ b/Environments/AKS-Store-Demo/abbreviations.json @@ -0,0 +1,136 @@ +{ + "analysisServicesServers": "as", + "apiManagementService": "apim-", + "appConfigurationConfigurationStores": "appcs-", + "appManagedEnvironments": "cae-", + "appContainerApps": "ca-", + "authorizationPolicyDefinitions": "policy-", + "automationAutomationAccounts": "aa-", + "blueprintBlueprints": "bp-", + "blueprintBlueprintsArtifacts": "bpa-", + "cacheRedis": "redis-", + "cdnProfiles": "cdnp-", + "cdnProfilesEndpoints": "cdne-", + "cognitiveServicesAccounts": "cog-", + "cognitiveServicesFormRecognizer": "cog-fr-", + "cognitiveServicesTextAnalytics": "cog-ta-", + "computeAvailabilitySets": "avail-", + "computeCloudServices": "cld-", + "computeDiskEncryptionSets": "des", + "computeDisks": "disk", + "computeDisksOs": "osdisk", + "computeGalleries": "gal", + "computeSnapshots": "snap-", + "computeVirtualMachines": "vm", + "computeVirtualMachineScaleSets": "vmss-", + "containerInstanceContainerGroups": "ci", + "containerRegistryRegistries": "cr", + "containerServiceManagedClusters": "aks-", + "databricksWorkspaces": "dbw-", + "dataFactoryFactories": "adf-", + "dataLakeAnalyticsAccounts": "dla", + "dataLakeStoreAccounts": "dls", + "dataMigrationServices": "dms-", + "dBforMySQLServers": "mysql-", + "dBforPostgreSQLServers": "psql-", + "devicesIotHubs": "iot-", + "devicesProvisioningServices": "provs-", + "devicesProvisioningServicesCertificates": "pcert-", + "documentDBDatabaseAccounts": "cosmos-", + "eventGridDomains": "evgd-", + "eventGridDomainsTopics": "evgt-", + "eventGridEventSubscriptions": "evgs-", + "eventHubNamespaces": "evhns-", + "eventHubNamespacesEventHubs": "evh-", + "hdInsightClustersHadoop": "hadoop-", + "hdInsightClustersHbase": "hbase-", + "hdInsightClustersKafka": "kafka-", + "hdInsightClustersMl": "mls-", + "hdInsightClustersSpark": "spark-", + "hdInsightClustersStorm": "storm-", + "hybridComputeMachines": "arcs-", + "insightsActionGroups": "ag-", + "insightsComponents": "appi-", + "keyVaultVaults": "kv-", + "kubernetesConnectedClusters": "arck", + "kustoClusters": "dec", + "kustoClustersDatabases": "dedb", + "loadTesting": "lt-", + "logicIntegrationAccounts": "ia-", + "logicWorkflows": "logic-", + "machineLearningServicesWorkspaces": "mlw-", + "managedIdentityUserAssignedIdentities": "id-", + "managementManagementGroups": "mg-", + "migrateAssessmentProjects": "migr-", + "networkApplicationGateways": "agw-", + "networkApplicationSecurityGroups": "asg-", + "networkAzureFirewalls": "afw-", + "networkBastionHosts": "bas-", + "networkConnections": "con-", + "networkDnsZones": "dnsz-", + "networkExpressRouteCircuits": "erc-", + "networkFirewallPolicies": "afwp-", + "networkFirewallPoliciesWebApplication": "waf", + "networkFirewallPoliciesRuleGroups": "wafrg", + "networkFrontDoors": "fd-", + "networkFrontdoorWebApplicationFirewallPolicies": "fdfp-", + "networkLoadBalancersExternal": "lbe-", + "networkLoadBalancersInternal": "lbi-", + "networkLoadBalancersInboundNatRules": "rule-", + "networkLocalNetworkGateways": "lgw-", + "networkNatGateways": "ng-", + "networkNetworkInterfaces": "nic-", + "networkNetworkSecurityGroups": "nsg-", + "networkNetworkSecurityGroupsSecurityRules": "nsgsr-", + "networkNetworkWatchers": "nw-", + "networkPrivateDnsZones": "pdnsz-", + "networkPrivateLinkServices": "pl-", + "networkPublicIPAddresses": "pip-", + "networkPublicIPPrefixes": "ippre-", + "networkRouteFilters": "rf-", + "networkRouteTables": "rt-", + "networkRouteTablesRoutes": "udr-", + "networkTrafficManagerProfiles": "traf-", + "networkVirtualNetworkGateways": "vgw-", + "networkVirtualNetworks": "vnet-", + "networkVirtualNetworksSubnets": "snet-", + "networkVirtualNetworksVirtualNetworkPeerings": "peer-", + "networkVirtualWans": "vwan-", + "networkVpnGateways": "vpng-", + "networkVpnGatewaysVpnConnections": "vcn-", + "networkVpnGatewaysVpnSites": "vst-", + "notificationHubsNamespaces": "ntfns-", + "notificationHubsNamespacesNotificationHubs": "ntf-", + "operationalInsightsWorkspaces": "log-", + "portalDashboards": "dash-", + "powerBIDedicatedCapacities": "pbi-", + "purviewAccounts": "pview-", + "recoveryServicesVaults": "rsv-", + "resourcesResourceGroups": "rg-", + "searchSearchServices": "srch-", + "serviceBusNamespaces": "sb-", + "serviceBusNamespacesQueues": "sbq-", + "serviceBusNamespacesTopics": "sbt-", + "serviceEndPointPolicies": "se-", + "serviceFabricClusters": "sf-", + "signalRServiceSignalR": "sigr", + "sqlManagedInstances": "sqlmi-", + "sqlServers": "sql-", + "sqlServersDataWarehouse": "sqldw-", + "sqlServersDatabases": "sqldb-", + "sqlServersDatabasesStretch": "sqlstrdb-", + "storageStorageAccounts": "st", + "storageStorageAccountsVm": "stvm", + "storSimpleManagers": "ssimp", + "streamAnalyticsCluster": "asa-", + "synapseWorkspaces": "syn", + "synapseWorkspacesAnalyticsWorkspaces": "synw", + "synapseWorkspacesSqlPoolsDedicated": "syndp", + "synapseWorkspacesSqlPoolsSpark": "synsp", + "timeSeriesInsightsEnvironments": "tsi-", + "webServerFarms": "plan-", + "webSitesAppService": "app-", + "webSitesAppServiceEnvironment": "ase-", + "webSitesFunctions": "func-", + "webStaticSites": "stapp-" +} \ No newline at end of file diff --git a/Environments/AKS-Store-Demo/app/aks-managed-cluster.bicep b/Environments/AKS-Store-Demo/app/aks-managed-cluster.bicep new file mode 100644 index 00000000..cff16f8d --- /dev/null +++ b/Environments/AKS-Store-Demo/app/aks-managed-cluster.bicep @@ -0,0 +1,136 @@ +metadata description = 'Creates an Azure Kubernetes Service (AKS) cluster with a system agent pool.' +@description('The name for the AKS managed cluster') +param name string + +@description('The name of the resource group for the managed resources of the AKS cluster') +param nodeResourceGroupName string = '' + +@description('The Azure region/location for the AKS resources') +param location string = resourceGroup().location + +@description('Custom tags to apply to the AKS resources') +param tags object = {} + +@description('Whether RBAC is enabled for local accounts') +param enableRbac bool = true + +// Add-ons +@description('Whether web app routing (preview) add-on is enabled') +param webAppRoutingAddon bool = true + +// AAD Integration +@description('Enable Azure Active Directory integration') +param enableAad bool = false + +@description('Enable RBAC using AAD') +param enableAzureRbac bool = false + +@description('The Tenant ID associated to the Azure Active Directory') +param aadTenantId string = '' + +@description('The load balancer SKU to use for ingress into the AKS cluster') +@allowed([ 'basic', 'standard' ]) +param loadBalancerSku string = 'standard' + +@description('Network plugin used for building the Kubernetes network.') +@allowed([ 'azure', 'kubenet', 'none' ]) +param networkPlugin string = 'azure' + +@description('If set to true, getting static credentials will be disabled for this cluster.') +param disableLocalAccounts bool = false + +@description('The managed cluster SKU.') +@allowed([ 'Free', 'Paid', 'Standard' ]) +param sku string = 'Free' + +@description('Configuration of AKS add-ons') +param addOns object = {} + +@description('The log analytics workspace id used for logging & monitoring') +param workspaceId string = '' + +@description('The node pool configuration for the System agent pool') +param systemPoolConfig object + +@description('The DNS prefix to associate with the AKS cluster') +param dnsPrefix string = '' + +resource aks 'Microsoft.ContainerService/managedClusters@2023-03-02-preview' = { + name: name + location: location + tags: tags + identity: { + type: 'SystemAssigned' + } + sku: { + name: 'Base' + tier: sku + } + properties: { + nodeResourceGroup: !empty(nodeResourceGroupName) ? nodeResourceGroupName : 'rg-mc-${name}' + dnsPrefix: empty(dnsPrefix) ? '${name}-dns' : dnsPrefix + enableRBAC: enableRbac + aadProfile: enableAad ? { + managed: true + enableAzureRBAC: enableAzureRbac + tenantID: aadTenantId + } : null + agentPoolProfiles: [ + systemPoolConfig + ] + networkProfile: { + loadBalancerSku: loadBalancerSku + networkPlugin: networkPlugin + } + disableLocalAccounts: disableLocalAccounts && enableAad + addonProfiles: addOns + securityProfile:{ + workloadIdentity: { + enabled: true + } + } + oidcIssuerProfile: { + enabled: true + } + } +} + +var aksDiagCategories = [ + 'cluster-autoscaler' + 'kube-controller-manager' + 'kube-audit-admin' + 'guard' +] + +// TODO: Update diagnostics to be its own module +// Blocking issue: https://github.com/Azure/bicep/issues/622 +// Unable to pass in a `resource` scope or unable to use string interpolation in resource types +resource diagnostics 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = if (!empty(workspaceId)) { + name: 'aks-diagnostics' + scope: aks + properties: { + workspaceId: workspaceId + logs: [for category in aksDiagCategories: { + category: category + enabled: true + }] + metrics: [ + { + category: 'AllMetrics' + enabled: true + } + ] + } +} + +@description('The resource name of the AKS cluster') +output clusterName string = aks.name + +@description('The AKS cluster identity') +output clusterIdentity object = { + clientId: aks.properties.identityProfile.kubeletidentity.clientId + objectId: aks.properties.identityProfile.kubeletidentity.objectId + resourceId: aks.properties.identityProfile.kubeletidentity.resourceId +} + +output clusterId string = aks.id diff --git a/Environments/AKS-Store-Demo/app/db.bicep b/Environments/AKS-Store-Demo/app/db.bicep new file mode 100644 index 00000000..9fe6e264 --- /dev/null +++ b/Environments/AKS-Store-Demo/app/db.bicep @@ -0,0 +1,55 @@ +@allowed([ + 'MongoDB' + 'GlobalDocumentDB' +]) +param kind string +param resourceToken string +param location string +param keyVaultName string +param tags object = {} +param cosmosDatabaseName string = 'orderdb' + +@description('The collections to create in the database') +param collections array = [ + { + id: 'orders' + name: 'orders' + shardKey: 'Hash' + indexKey: '_id' + throughput: 400 + } +] + +// the application database +module cosmosMongo '../core/database/cosmos/mongo/cosmos-mongo-db.bicep' = if(kind == 'MongoDB') { + name: 'cosmos-mongo' + params: { + accountName: 'cosmos-${resourceToken}' + databaseName: cosmosDatabaseName + location: location + collections: collections + tags: tags + keyVaultName: keyVaultName + } +} + +module cosmosSql '../core/database/cosmos/sql/cosmos-sql-db.bicep' = if(kind == 'GlobalDocumentDB') { + name: 'cosmos-sql' + params: { + accountName: 'cosmos-${resourceToken}' + databaseName: cosmosDatabaseName + location: location + containers: [ + { + name: 'orders' + id: 'orders' + partitionKey: '/storeId' + } + ] + tags: tags + keyVaultName: keyVaultName + } +} + +output name string = 'cosmos-${resourceToken}' +output endpoint string = kind == 'MongoDB' ? 'mongodb://cosmos-${resourceToken}.mongo.cosmos.azure.com:10255/?retryWrites=false' : 'https://cosmos-${resourceToken}.documents.azure.com:443/' diff --git a/Environments/AKS-Store-Demo/app/get-keys.bicep b/Environments/AKS-Store-Demo/app/get-keys.bicep new file mode 100644 index 00000000..ea501e5d --- /dev/null +++ b/Environments/AKS-Store-Demo/app/get-keys.bicep @@ -0,0 +1,35 @@ +param openAiName string +param openAiKeyName string = 'AZURE-OPENAI-KEY' +param cosmosAccountName string +param cosmosKeyName string = 'AZURE-COSMOS-KEY' +param keyVaultName string + +resource account 'Microsoft.CognitiveServices/accounts@2023-05-01' existing = { + name: openAiName +} + +resource cosmos 'Microsoft.DocumentDB/databaseAccounts@2022-08-15' existing = { + name: cosmosAccountName +} + +// create key vault secrets +module openAiKey '../core/security/keyvault-secret.bicep' = { + name: 'openAiKey' + params: { + name: openAiKeyName + keyVaultName: keyVaultName + secretValue: account.listKeys().key1 + } +} + +module cosmosKey '../core/security/keyvault-secret.bicep' = { + name: 'cosmosKey' + params: { + name: cosmosKeyName + keyVaultName: keyVaultName + secretValue: cosmos.listKeys().primaryMasterKey + } +} + +output openAiKey string = openAiKeyName +output cosmosKey string = cosmosKeyName diff --git a/Environments/AKS-Store-Demo/app/identity.bicep b/Environments/AKS-Store-Demo/app/identity.bicep new file mode 100644 index 00000000..71667acd --- /dev/null +++ b/Environments/AKS-Store-Demo/app/identity.bicep @@ -0,0 +1,53 @@ +param name string +param location string +param tags object = {} +param principalId string +param AZURE_AKS_NAMESPACE string +param clusterName string + +// identity for the openai +resource identity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { + name: name + location: location + tags: tags +} + +// federated credential for the openai +resource federatedCredential 'Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials@2023-01-31' = { + name: name + parent: identity + properties: { + audiences: ['api://AzureADTokenExchange'] + issuer: aks.properties.oidcIssuerProfile.issuerURL + subject: 'system:serviceaccount:${AZURE_AKS_NAMESPACE}:ai-service-account' + } +} + +// role definition for the openai +var openAiUserRole = '5e0bd9bd-7b93-4f28-af87-19fc36ad61bd' + +// role assignment for the openai +module roleAssignment '../core/security/role.bicep' = { + name: 'roleAssignment' + params: { + principalId: identity.properties.principalId + principalType: 'ServicePrincipal' + roleDefinitionId: openAiUserRole + } +} + +module roleAssignmentForMe '../core/security/role.bicep' = { + name: 'roleAssignmentForMe' + params: { + principalId: principalId + principalType: 'User' + roleDefinitionId: openAiUserRole + } +} + +resource aks 'Microsoft.ContainerService/managedClusters@2023-03-02-preview' existing = { + name: clusterName +} + +output principalId string = identity.properties.principalId +output clientId string = identity.properties.clientId diff --git a/Environments/AKS-Store-Demo/app/monitor.bicep b/Environments/AKS-Store-Demo/app/monitor.bicep new file mode 100644 index 00000000..c0cd74bc --- /dev/null +++ b/Environments/AKS-Store-Demo/app/monitor.bicep @@ -0,0 +1,12 @@ +param name string +param location string +param tags object = {} + +resource monitor 'Microsoft.Monitor/accounts@2023-04-03' = { + name: name + location: location + tags: tags +} + +output id string = monitor.id +output name string = monitor.name diff --git a/Environments/AKS-Store-Demo/app/observability.bicep b/Environments/AKS-Store-Demo/app/observability.bicep new file mode 100644 index 00000000..671d06cd --- /dev/null +++ b/Environments/AKS-Store-Demo/app/observability.bicep @@ -0,0 +1,327 @@ +param name string +param location string +param tags object = {} + +param monitorName string +param monitorId string +param principalId string + +param clusterId string +param clusterName string + +param logAnalyticsId string +param logAnalyticsName string + +resource grafana 'Microsoft.Dashboard/grafana@2022-08-01' = { + name: name + location: location + tags: tags + identity: { + type: 'SystemAssigned' + } + properties: { + grafanaIntegrations: { + azureMonitorWorkspaceIntegrations: [ + { + azureMonitorWorkspaceResourceId: monitorId + } + ] + } + } + sku: { + name: 'Standard' + } +} + +var grafanaAdminRole = '22926164-76b3-42b3-bc55-97df8dab3e41' +var monitorReaderRole = 'b0d8363b-8ddd-447d-831f-62ca05bff136' + +// role assignment for the grafana +module roleAssignmentForMe '../core/security/role.bicep' = { + name: 'grafanaRoleAssignmentForMe' + params: { + principalId: principalId + principalType: 'User' + roleDefinitionId: grafanaAdminRole + } +} + +module roleAssignment '../core/security/role.bicep' = { + name: 'monitorRoleAssignmentForGrafana' + params: { + principalId: grafana.identity.principalId + principalType: 'ServicePrincipal' + roleDefinitionId: monitorReaderRole + } +} + +// data collection +resource dataCollectionEndpoint 'Microsoft.Insights/dataCollectionEndpoints@2022-06-01' = { + name: 'MSProm-${clusterName}' + location: location + kind: 'Linux' + properties:{} +} + +resource dataCollectionRule 'Microsoft.Insights/dataCollectionRules@2022-06-01' = { + name: 'MSProm-${clusterName}' + location: location + properties: { + dataCollectionEndpointId: dataCollectionEndpoint.id + dataSources:{ + prometheusForwarder: [ + { + name: 'PrometheusDataSource' + streams: ['Microsoft-PrometheusMetrics'] + } + ] + } + destinations: { + monitoringAccounts: [ + { + accountResourceId: monitorId + name: monitorName + } + ] + } + dataFlows: [ + { + streams: ['Microsoft-PrometheusMetrics'] + destinations:[ + monitorName + ] + } + ] + } +} + +resource aks 'Microsoft.ContainerService/managedClusters@2023-03-02-preview' existing = { + name: clusterName +} + +resource dcrToAks 'Microsoft.Insights/dataCollectionRuleAssociations@2022-06-01' = { + name: 'dcr-${clusterName}' + scope: aks + properties: { + dataCollectionRuleId: dataCollectionRule.id + } +} + +resource dcrToAksEndpoint 'Microsoft.Insights/dataCollectionRuleAssociations@2022-06-01' = { + name: 'configurationAccessEndpoint' + scope: aks + properties: { + dataCollectionEndpointId: dataCollectionEndpoint.id + } +} + +resource monitorPrometheusRuleGroup 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'NodeRecordingRulesRuleGroup-${clusterName}' + location: location + properties:{ + interval: 'PT1M' + enabled: true + clusterName: clusterName + scopes:[monitorId] + rules:[ + { + record: 'instance:node_num_cpu:sum' + expression: 'count without (cpu, mode) (node_cpu_seconds_total{job="node",mode="idle"})' + } + { + record: 'instance:node_cpu_utilisation:rate5m' + expression: '1 - avg without (cpu) (sum without (mode) (rate(node_cpu_seconds_total{job="node", mode="idle"}[5m])) + sum without (mode) (rate(node_cpu_seconds_total{job="node", mode="iowait"}[5m])) + sum without (mode) (rate(node_cpu_seconds_total{job="node", mode="steal"}[5m])))' + } + { + record: 'instance:node_load1_per_cpu:ratio' + expression: 'node_load1{job="node"}/instance:node_num_cpu:sum{job="node"}' + } + { + record: 'instance:node_memory_utilisation:ratio' + expression: '1 - ((node_memory_MemAvailable_bytes{job="node"} or (node_memory_Buffers_bytes{job="node"} + node_memory_Cached_bytes{job="node"} + node_memory_MemFree_bytes{job="node"} + node_memory_Slab_bytes{job="node"})) / node_memory_MemTotal_bytes{job="node"})' + } + { + record: 'instance:node_vmstat_pgmajfault:rate5m' + expression: 'rate(node_vmstat_pgmajfault{job="node"}[5m])' + } + { + record: 'instance_device:node_disk_io_time_seconds:rate5m' + expression: 'rate(node_disk_io_time_seconds_total{job="node", device!=""}[5m])' + } + { + record: 'instance_device:node_disk_io_time_weighted_seconds:rate5m' + expression: 'rate(node_disk_io_time_weighted_seconds_total{job="node", device!=""}[5m])' + } + { + record: 'instance:node_network_receive_bytes_excluding_lo:rate5m' + expression: 'sum without (device) (rate(node_network_receive_bytes_total{job="node", device!="lo"}[5m]))' + } + { + record: 'instance:node_network_transmit_bytes_excluding_lo:rate5m' + expression: 'sum without (device) (rate(node_network_transmit_bytes_total{job="node", device!="lo"}[5m]))' + } + { + record: 'instance:node_network_receive_drop_excluding_lo:rate5m' + expression: 'sum without (device) (rate(node_network_receive_drop_total{job="node", device!="lo"}[5m]))' + } + { + record: 'instance:node_network_transmit_drop_excluding_lo:rate5m' + expression: 'sum without (device) (rate(node_network_transmit_drop_total{job="node", device!="lo"}[5m]))' + } + ] + } +} + +resource k8sPrometheusRuleGroup 'Microsoft.AlertsManagement/prometheusRuleGroups@2023-03-01' = { + name: 'KubernetesRecordingRulesRuleGroup-${clusterName}' + location: location + properties:{ + interval: 'PT1M' + enabled: true + clusterName: clusterName + scopes:[monitorId] + rules:[ + { + record: 'node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate' + expression: 'sum by (cluster, namespace, pod, container) (irate(container_cpu_usage_seconds_total{job="cadvisor", image!=""}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}))' + } + { + record: 'node_namespace_pod_container:container_memory_working_set_bytes' + expression: 'container_memory_working_set_bytes{job="cadvisor", image!=""}* on (namespace, pod) group_left(node) topk by(namespace, pod) (1, max by(namespace, pod, node) (kube_pod_info{node!=""}))' + } + { + record: 'node_namespace_pod_container:container_memory_rss' + expression: 'container_memory_rss{job="cadvisor", image!=""}* on (namespace, pod) group_left(node) topk by(namespace, pod) (1, max by(namespace, pod, node) (kube_pod_info{node!=""}))' + } + { + record: 'node_namespace_pod_container:container_memory_cache' + expression: 'container_memory_cache{job="cadvisor", image!=""}* on (namespace, pod) group_left(node) topk by(namespace, pod) (1, max by(namespace, pod, node) (kube_pod_info{node!=""}))' + } + { + record: 'node_namespace_pod_container:container_memory_swap' + expression: 'container_memory_swap{job="cadvisor", image!=""}* on (namespace, pod) group_left(node) topk by(namespace, pod) (1, max by(namespace, pod, node) (kube_pod_info{node!=""}))' + } + { + record: 'cluster:namespace:pod_memory:active:kube_pod_container_resource_requests' + expression: 'kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} * on(namespace, pod, cluster)group_left() max by (namespace, pod, cluster) ((kube_pod_status_phase{phase=~"Pending|Running"} == 1))' + } + { + record: 'namespace_memory:kube_pod_container_resource_requests:sum' + expression: 'sum by (namespace, cluster) (sum by (namespace, pod, cluster) (max by (namespace, pod, container, cluster) (kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"}) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (kube_pod_status_phase{phase=~"Pending|Running"} == 1)))' + } + { + record: 'cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests' + expression: 'kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster)group_left() max by (namespace, pod, cluster) ((kube_pod_status_phase{phase=~"Pending|Running"} == 1))' + } + { + record: 'namespace_cpu:kube_pod_container_resource_requests:sum' + expression: 'sum by (namespace, cluster) (sum by(namespace, pod, cluster) (max by(namespace, pod, container, cluster) (kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"}) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (kube_pod_status_phase{phase=~"Pending|Running"} == 1)))' + } + { + record: 'cluster:namespace:pod_memory:active:kube_pod_container_resource_limits' + expression: 'kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster)group_left() max by (namespace, pod, cluster) ((kube_pod_status_phase{phase=~"Pending|Running"} == 1))' + } + { + record: 'namespace_memory:kube_pod_container_resource_limits:sum' + expression: 'sum by (namespace, cluster) (sum by (namespace, pod, cluster) (max by (namespace, pod, container, cluster) (kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"}) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (kube_pod_status_phase{phase=~"Pending|Running"} == 1)))' + } + { + record: 'cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits' + expression: 'kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster)group_left() max by (namespace, pod, cluster) ((kube_pod_status_phase{phase=~"Pending|Running"} == 1))' + } + { + record: 'namespace_cpu:kube_pod_container_resource_limits:sum' + expression: 'sum by (namespace, cluster) (sum by (namespace, pod, cluster) (max by(namespace, pod, container, cluster) (kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"}) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (kube_pod_status_phase{phase=~"Pending|Running"} == 1)))' + } + { + record: 'namespace_workload_pod:kube_pod_owner:relabel' + expression: 'max by (cluster, namespace, workload, pod) (label_replace(label_replace(kube_pod_owner{job="kube-state-metrics", owner_kind="ReplicaSet"}, "replicaset", "$1", "owner_name", "(.*)") * on(replicaset, namespace) group_left(owner_name) topk by(replicaset, namespace) (1, max by (replicaset, namespace, owner_name) (kube_replicaset_owner{job="kube-state-metrics"})), "workload", "$1", "owner_name", "(.*)"))' + labels: { + workload_type: 'deployment' + } + } + { + record: 'namespace_workload_pod:kube_pod_owner:relabel' + expression: 'max by (cluster, namespace, workload, pod) (label_replace(kube_pod_owner{job="kube-state-metrics", owner_kind="DaemonSet"}, "workload", "$1", "owner_name", "(.*)"))' + labels: { + workload_type: 'daemonset' + } + } + { + record: 'namespace_workload_pod:kube_pod_owner:relabel' + expression: 'max by (cluster, namespace, workload, pod) (label_replace(kube_pod_owner{job="kube-state-metrics", owner_kind="StatefulSet"}, "workload", "$1", "owner_name", "(.*)"))' + labels: { + workload_type: 'statefulset' + } + } + { + record: 'namespace_workload_pod:kube_pod_owner:relabel' + expression: 'max by (cluster, namespace, workload, pod) (label_replace(kube_pod_owner{job="kube-state-metrics", owner_kind="Job"}, "workload", "$1", "owner_name", "(.*)"))' + labels: { + workload_type: 'job' + } + } + { + record: ':node_memory_MemAvailable_bytes:sum' + expression: 'sum(node_memory_MemAvailable_bytes{job="node"} or (node_memory_Buffers_bytes{job="node"} + node_memory_Cached_bytes{job="node"} + node_memory_MemFree_bytes{job="node"} + node_memory_Slab_bytes{job="node"})) by (cluster)' + } + { + record: 'cluster:node_cpu:ratio_rate5m' + expression: 'sum(rate(node_cpu_seconds_total{job="node",mode!="idle",mode!="iowait",mode!="steal"}[5m])) by (cluster) /count(sum(node_cpu_seconds_total{job="node"}) by (cluster, instance, cpu)) by (cluster)' + } + ] + } +} + +resource monitorDataCollectionRule 'Microsoft.Insights/dataCollectionRules@2022-06-01' = { + name: 'MSCI-${clusterName}' + location: location + kind: 'Linux' + properties: { + dataSources: { + extensions: [ + { + streams: [ + 'Microsoft-ContainerInsights-Group-Default' + ] + extensionName: 'ContainerInsights' + extensionSettings: { + dataCollectionSettings: { + enableContainerLogV2: true + interval: '1m' + namespaceFilteringMode: 'Off' + } + } + name: 'ContainerInsightsExtension' + } + ] + } + destinations: { + logAnalytics:[ + { + workspaceResourceId: logAnalyticsId + name: logAnalyticsName + } + ] + } + dataFlows: [ + { + streams: [ + 'Microsoft-ContainerInsights-Group-Default' + ] + destinations: [ + logAnalyticsName + ] + } + ] + } +} + + +resource msciToAks 'Microsoft.Insights/dataCollectionRuleAssociations@2022-06-01' = { + name: 'msci-${clusterName}' + scope: aks + properties: { + dataCollectionRuleId: dataCollectionRule.id + } +} diff --git a/Environments/AKS-Store-Demo/app/servicebus.bicep b/Environments/AKS-Store-Demo/app/servicebus.bicep new file mode 100644 index 00000000..778485d4 --- /dev/null +++ b/Environments/AKS-Store-Demo/app/servicebus.bicep @@ -0,0 +1,70 @@ +param name string +param location string +param tags object = {} + +param keyVaultName string +param listenerKeyName string = 'AZURE-SERVICE-BUS-LISTENER-KEY' +param senderKeyName string = 'AZURE-SERVICE-BUS-SENDER-KEY' + +// Service Bus Namespace +resource serviceBusNamespace 'Microsoft.ServiceBus/namespaces@2022-01-01-preview' = { + name: name + location: location + tags: tags + sku: { + name: 'Standard' + } +} + +// Service Bus Namespace Authorization Rule +resource serviceBusNamespaceAuthorizationRule 'Microsoft.ServiceBus/namespaces/AuthorizationRules@2022-10-01-preview' = { + name: 'listener' + parent: serviceBusNamespace + properties: { + rights: [ + 'Listen' + ] + } +} + +// Service Bus Queue +resource serviceBusQueue 'Microsoft.ServiceBus/namespaces/queues@2022-01-01-preview' = { + name: 'orders' + parent: serviceBusNamespace +} + +// Service Bus Namespace Authorization Rule +resource serviceBusNamespaceQueueAuthorizationRule 'Microsoft.ServiceBus/namespaces/AuthorizationRules@2022-10-01-preview' = { + name: 'sender' + parent: serviceBusNamespace + properties: { + rights: [ + 'Send' + ] + } +} + +module senderKey '../core/security/keyvault-secret.bicep' = { + name: 'senderKey' + params: { + name: senderKeyName + keyVaultName: keyVaultName + secretValue: serviceBusNamespaceQueueAuthorizationRule.listKeys().primaryKey + } +} + +module listenerKey '../core/security/keyvault-secret.bicep' = { + name: 'listenerKey' + params: { + name: listenerKeyName + keyVaultName: keyVaultName + secretValue: serviceBusNamespaceAuthorizationRule.listKeys().primaryKey + } +} + +output serviceBusEndpoint string = serviceBusNamespace.properties.serviceBusEndpoint +output serviceBusListenerName string = serviceBusNamespaceAuthorizationRule.name +output serviceBusSenderName string = serviceBusNamespaceQueueAuthorizationRule.name +output serviceBusListenerKey string = listenerKeyName +output serviceBusSenderKey string = senderKeyName +output serviceBusNamespaceName string = serviceBusNamespace.name diff --git a/Environments/AKS-Store-Demo/core/ai/cognitiveservices.bicep b/Environments/AKS-Store-Demo/core/ai/cognitiveservices.bicep new file mode 100644 index 00000000..1bf5666b --- /dev/null +++ b/Environments/AKS-Store-Demo/core/ai/cognitiveservices.bicep @@ -0,0 +1,53 @@ +metadata description = 'Creates an Azure Cognitive Services instance.' +param name string +param location string = resourceGroup().location +param tags object = {} +@description('The custom subdomain name used to access the API. Defaults to the value of the name parameter.') +param customSubDomainName string = name +param deployments array = [] +param kind string = 'OpenAI' + +@allowed([ 'Enabled', 'Disabled' ]) +param publicNetworkAccess string = 'Enabled' +param sku object = { + name: 'S0' +} + +param allowedIpRules array = [] +param networkAcls object = empty(allowedIpRules) ? { + defaultAction: 'Allow' +} : { + ipRules: allowedIpRules + defaultAction: 'Deny' +} + +resource account 'Microsoft.CognitiveServices/accounts@2023-05-01' = { + name: name + location: location + tags: tags + kind: kind + properties: { + customSubDomainName: customSubDomainName + publicNetworkAccess: publicNetworkAccess + networkAcls: networkAcls + } + sku: sku +} + +@batchSize(1) +resource deployment 'Microsoft.CognitiveServices/accounts/deployments@2023-05-01' = [for deployment in deployments: { + parent: account + name: deployment.name + properties: { + model: deployment.model + raiPolicyName: contains(deployment, 'raiPolicyName') ? deployment.raiPolicyName : null + } + sku: contains(deployment, 'sku') ? deployment.sku : { + name: 'Standard' + capacity: 20 + } +}] + +output endpoint string = account.properties.endpoint +output id string = account.id +output name string = account.name diff --git a/Environments/AKS-Store-Demo/core/database/cosmos/cosmos-account.bicep b/Environments/AKS-Store-Demo/core/database/cosmos/cosmos-account.bicep new file mode 100644 index 00000000..6f8747f5 --- /dev/null +++ b/Environments/AKS-Store-Demo/core/database/cosmos/cosmos-account.bicep @@ -0,0 +1,49 @@ +metadata description = 'Creates an Azure Cosmos DB account.' +param name string +param location string = resourceGroup().location +param tags object = {} + +param connectionStringKey string = 'AZURE-COSMOS-CONNECTION-STRING' +param keyVaultName string + +@allowed([ 'GlobalDocumentDB', 'MongoDB', 'Parse' ]) +param kind string + +resource cosmos 'Microsoft.DocumentDB/databaseAccounts@2022-08-15' = { + name: name + kind: kind + location: location + tags: tags + properties: { + consistencyPolicy: { defaultConsistencyLevel: 'Session' } + locations: [ + { + locationName: location + failoverPriority: 0 + isZoneRedundant: false + } + ] + databaseAccountOfferType: 'Standard' + enableAutomaticFailover: false + enableMultipleWriteLocations: false + apiProperties: (kind == 'MongoDB') ? { serverVersion: '4.2' } : {} + capabilities: [ { name: 'EnableServerless' } ] + } +} + +resource cosmosConnectionString 'Microsoft.KeyVault/vaults/secrets@2022-07-01' = { + parent: keyVault + name: connectionStringKey + properties: { + value: cosmos.listConnectionStrings().connectionStrings[0].connectionString + } +} + +resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' existing = { + name: keyVaultName +} + +output connectionStringKey string = connectionStringKey +output endpoint string = cosmos.properties.documentEndpoint +output id string = cosmos.id +output name string = cosmos.name diff --git a/Environments/AKS-Store-Demo/core/database/cosmos/mongo/cosmos-mongo-account.bicep b/Environments/AKS-Store-Demo/core/database/cosmos/mongo/cosmos-mongo-account.bicep new file mode 100644 index 00000000..4aafbf38 --- /dev/null +++ b/Environments/AKS-Store-Demo/core/database/cosmos/mongo/cosmos-mongo-account.bicep @@ -0,0 +1,23 @@ +metadata description = 'Creates an Azure Cosmos DB for MongoDB account.' +param name string +param location string = resourceGroup().location +param tags object = {} + +param keyVaultName string +param connectionStringKey string = 'AZURE-COSMOS-CONNECTION-STRING' + +module cosmos '../../cosmos/cosmos-account.bicep' = { + name: 'cosmos-account' + params: { + name: name + location: location + connectionStringKey: connectionStringKey + keyVaultName: keyVaultName + kind: 'MongoDB' + tags: tags + } +} + +output connectionStringKey string = cosmos.outputs.connectionStringKey +output endpoint string = cosmos.outputs.endpoint +output id string = cosmos.outputs.id diff --git a/Environments/AKS-Store-Demo/core/database/cosmos/mongo/cosmos-mongo-db.bicep b/Environments/AKS-Store-Demo/core/database/cosmos/mongo/cosmos-mongo-db.bicep new file mode 100644 index 00000000..2a670578 --- /dev/null +++ b/Environments/AKS-Store-Demo/core/database/cosmos/mongo/cosmos-mongo-db.bicep @@ -0,0 +1,47 @@ +metadata description = 'Creates an Azure Cosmos DB for MongoDB account with a database.' +param accountName string +param databaseName string +param location string = resourceGroup().location +param tags object = {} + +param collections array = [] +param connectionStringKey string = 'AZURE-COSMOS-CONNECTION-STRING' +param keyVaultName string + +module cosmos 'cosmos-mongo-account.bicep' = { + name: 'cosmos-mongo-account' + params: { + name: accountName + location: location + keyVaultName: keyVaultName + tags: tags + connectionStringKey: connectionStringKey + } +} + +resource database 'Microsoft.DocumentDB/databaseAccounts/mongodbDatabases@2022-08-15' = { + name: '${accountName}/${databaseName}' + tags: tags + properties: { + resource: { id: databaseName } + } + + resource list 'collections' = [for collection in collections: { + name: collection.name + properties: { + resource: { + id: collection.id + shardKey: { _id: collection.shardKey } + indexes: [ { key: { keys: [ collection.indexKey ] } } ] + } + } + }] + + dependsOn: [ + cosmos + ] +} + +output connectionStringKey string = connectionStringKey +output databaseName string = databaseName +output endpoint string = cosmos.outputs.endpoint diff --git a/Environments/AKS-Store-Demo/core/database/cosmos/sql/cosmos-sql-account.bicep b/Environments/AKS-Store-Demo/core/database/cosmos/sql/cosmos-sql-account.bicep new file mode 100644 index 00000000..8431135e --- /dev/null +++ b/Environments/AKS-Store-Demo/core/database/cosmos/sql/cosmos-sql-account.bicep @@ -0,0 +1,22 @@ +metadata description = 'Creates an Azure Cosmos DB for NoSQL account.' +param name string +param location string = resourceGroup().location +param tags object = {} + +param keyVaultName string + +module cosmos '../../cosmos/cosmos-account.bicep' = { + name: 'cosmos-account' + params: { + name: name + location: location + tags: tags + keyVaultName: keyVaultName + kind: 'GlobalDocumentDB' + } +} + +output connectionStringKey string = cosmos.outputs.connectionStringKey +output endpoint string = cosmos.outputs.endpoint +output id string = cosmos.outputs.id +output name string = cosmos.outputs.name diff --git a/Environments/AKS-Store-Demo/core/database/cosmos/sql/cosmos-sql-db.bicep b/Environments/AKS-Store-Demo/core/database/cosmos/sql/cosmos-sql-db.bicep new file mode 100644 index 00000000..265880dc --- /dev/null +++ b/Environments/AKS-Store-Demo/core/database/cosmos/sql/cosmos-sql-db.bicep @@ -0,0 +1,74 @@ +metadata description = 'Creates an Azure Cosmos DB for NoSQL account with a database.' +param accountName string +param databaseName string +param location string = resourceGroup().location +param tags object = {} + +param containers array = [] +param keyVaultName string +param principalIds array = [] + +module cosmos 'cosmos-sql-account.bicep' = { + name: 'cosmos-sql-account' + params: { + name: accountName + location: location + tags: tags + keyVaultName: keyVaultName + } +} + +resource database 'Microsoft.DocumentDB/databaseAccounts/sqlDatabases@2022-05-15' = { + name: '${accountName}/${databaseName}' + properties: { + resource: { id: databaseName } + } + + resource list 'containers' = [for container in containers: { + name: container.name + properties: { + resource: { + id: container.id + partitionKey: { paths: [ container.partitionKey ] } + } + options: {} + } + }] + + dependsOn: [ + cosmos + ] +} + +module roleDefinition 'cosmos-sql-role-def.bicep' = { + name: 'cosmos-sql-role-definition' + params: { + accountName: accountName + } + dependsOn: [ + cosmos + database + ] +} + +// We need batchSize(1) here because sql role assignments have to be done sequentially +@batchSize(1) +module userRole 'cosmos-sql-role-assign.bicep' = [for principalId in principalIds: if (!empty(principalId)) { + name: 'cosmos-sql-user-role-${uniqueString(principalId)}' + params: { + accountName: accountName + roleDefinitionId: roleDefinition.outputs.id + principalId: principalId + } + dependsOn: [ + cosmos + database + ] +}] + +output accountId string = cosmos.outputs.id +output accountName string = cosmos.outputs.name +output connectionStringKey string = cosmos.outputs.connectionStringKey +output databaseName string = databaseName +output endpoint string = cosmos.outputs.endpoint +output roleDefinitionId string = roleDefinition.outputs.id diff --git a/Environments/AKS-Store-Demo/core/database/cosmos/sql/cosmos-sql-role-assign.bicep b/Environments/AKS-Store-Demo/core/database/cosmos/sql/cosmos-sql-role-assign.bicep new file mode 100644 index 00000000..3949efef --- /dev/null +++ b/Environments/AKS-Store-Demo/core/database/cosmos/sql/cosmos-sql-role-assign.bicep @@ -0,0 +1,19 @@ +metadata description = 'Creates a SQL role assignment under an Azure Cosmos DB account.' +param accountName string + +param roleDefinitionId string +param principalId string = '' + +resource role 'Microsoft.DocumentDB/databaseAccounts/sqlRoleAssignments@2022-05-15' = { + parent: cosmos + name: guid(roleDefinitionId, principalId, cosmos.id) + properties: { + principalId: principalId + roleDefinitionId: roleDefinitionId + scope: cosmos.id + } +} + +resource cosmos 'Microsoft.DocumentDB/databaseAccounts@2022-08-15' existing = { + name: accountName +} diff --git a/Environments/AKS-Store-Demo/core/database/cosmos/sql/cosmos-sql-role-def.bicep b/Environments/AKS-Store-Demo/core/database/cosmos/sql/cosmos-sql-role-def.bicep new file mode 100644 index 00000000..778d6dc4 --- /dev/null +++ b/Environments/AKS-Store-Demo/core/database/cosmos/sql/cosmos-sql-role-def.bicep @@ -0,0 +1,30 @@ +metadata description = 'Creates a SQL role definition under an Azure Cosmos DB account.' +param accountName string + +resource roleDefinition 'Microsoft.DocumentDB/databaseAccounts/sqlRoleDefinitions@2022-08-15' = { + parent: cosmos + name: guid(cosmos.id, accountName, 'sql-role') + properties: { + assignableScopes: [ + cosmos.id + ] + permissions: [ + { + dataActions: [ + 'Microsoft.DocumentDB/databaseAccounts/readMetadata' + 'Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers/items/*' + 'Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers/*' + ] + notDataActions: [] + } + ] + roleName: 'Reader Writer' + type: 'CustomRole' + } +} + +resource cosmos 'Microsoft.DocumentDB/databaseAccounts@2022-08-15' existing = { + name: accountName +} + +output id string = roleDefinition.id diff --git a/Environments/AKS-Store-Demo/core/host/container-registry.bicep b/Environments/AKS-Store-Demo/core/host/container-registry.bicep new file mode 100644 index 00000000..9c64531b --- /dev/null +++ b/Environments/AKS-Store-Demo/core/host/container-registry.bicep @@ -0,0 +1,83 @@ +metadata description = 'Creates an Azure Container Registry.' +param name string +param location string = resourceGroup().location +param tags object = {} + +@description('Indicates whether admin user is enabled') +param adminUserEnabled bool = false + +@description('Indicates whether anonymous pull is enabled') +param anonymousPullEnabled bool = false + +@description('Indicates whether data endpoint is enabled') +param dataEndpointEnabled bool = false + +@description('Encryption settings') +param encryption object = { + status: 'disabled' +} + +@description('Options for bypassing network rules') +param networkRuleBypassOptions string = 'AzureServices' + +@description('Public network access setting') +param publicNetworkAccess string = 'Enabled' + +@description('SKU settings') +param sku object = { + name: 'Basic' +} + +@description('Zone redundancy setting') +param zoneRedundancy string = 'Disabled' + +@description('The log analytics workspace ID used for logging and monitoring') +param workspaceId string = '' + +// 2022-02-01-preview needed for anonymousPullEnabled +resource containerRegistry 'Microsoft.ContainerRegistry/registries@2022-02-01-preview' = { + name: name + location: location + tags: tags + sku: sku + properties: { + adminUserEnabled: adminUserEnabled + anonymousPullEnabled: anonymousPullEnabled + dataEndpointEnabled: dataEndpointEnabled + encryption: encryption + networkRuleBypassOptions: networkRuleBypassOptions + publicNetworkAccess: publicNetworkAccess + zoneRedundancy: zoneRedundancy + } +} + +// TODO: Update diagnostics to be its own module +// Blocking issue: https://github.com/Azure/bicep/issues/622 +// Unable to pass in a `resource` scope or unable to use string interpolation in resource types +resource diagnostics 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = if (!empty(workspaceId)) { + name: 'registry-diagnostics' + scope: containerRegistry + properties: { + workspaceId: workspaceId + logs: [ + { + category: 'ContainerRegistryRepositoryEvents' + enabled: true + } + { + category: 'ContainerRegistryLoginEvents' + enabled: true + } + ] + metrics: [ + { + category: 'AllMetrics' + enabled: true + timeGrain: 'PT1M' + } + ] + } +} + +output loginServer string = containerRegistry.properties.loginServer +output name string = containerRegistry.name diff --git a/Environments/AKS-Store-Demo/core/monitor/loganalytics.bicep b/Environments/AKS-Store-Demo/core/monitor/loganalytics.bicep new file mode 100644 index 00000000..33f9dc29 --- /dev/null +++ b/Environments/AKS-Store-Demo/core/monitor/loganalytics.bicep @@ -0,0 +1,22 @@ +metadata description = 'Creates a Log Analytics workspace.' +param name string +param location string = resourceGroup().location +param tags object = {} + +resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' = { + name: name + location: location + tags: tags + properties: any({ + retentionInDays: 30 + features: { + searchVersion: 1 + } + sku: { + name: 'PerGB2018' + } + }) +} + +output id string = logAnalytics.id +output name string = logAnalytics.name diff --git a/Environments/AKS-Store-Demo/core/security/keyvault-access.bicep b/Environments/AKS-Store-Demo/core/security/keyvault-access.bicep new file mode 100644 index 00000000..316775f2 --- /dev/null +++ b/Environments/AKS-Store-Demo/core/security/keyvault-access.bicep @@ -0,0 +1,22 @@ +metadata description = 'Assigns an Azure Key Vault access policy.' +param name string = 'add' + +param keyVaultName string +param permissions object = { secrets: [ 'get', 'list' ] } +param principalId string + +resource keyVaultAccessPolicies 'Microsoft.KeyVault/vaults/accessPolicies@2022-07-01' = { + parent: keyVault + name: name + properties: { + accessPolicies: [ { + objectId: principalId + tenantId: subscription().tenantId + permissions: permissions + } ] + } +} + +resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' existing = { + name: keyVaultName +} diff --git a/Environments/AKS-Store-Demo/core/security/keyvault-secret.bicep b/Environments/AKS-Store-Demo/core/security/keyvault-secret.bicep new file mode 100644 index 00000000..7441b296 --- /dev/null +++ b/Environments/AKS-Store-Demo/core/security/keyvault-secret.bicep @@ -0,0 +1,31 @@ +metadata description = 'Creates or updates a secret in an Azure Key Vault.' +param name string +param tags object = {} +param keyVaultName string +param contentType string = 'string' +@description('The value of the secret. Provide only derived values like blob storage access, but do not hard code any secrets in your templates') +@secure() +param secretValue string + +param enabled bool = true +param exp int = 0 +param nbf int = 0 + +resource keyVaultSecret 'Microsoft.KeyVault/vaults/secrets@2022-07-01' = { + name: name + tags: tags + parent: keyVault + properties: { + attributes: { + enabled: enabled + exp: exp + nbf: nbf + } + contentType: contentType + value: secretValue + } +} + +resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' existing = { + name: keyVaultName +} diff --git a/Environments/AKS-Store-Demo/core/security/keyvault.bicep b/Environments/AKS-Store-Demo/core/security/keyvault.bicep new file mode 100644 index 00000000..314a1db6 --- /dev/null +++ b/Environments/AKS-Store-Demo/core/security/keyvault.bicep @@ -0,0 +1,26 @@ +metadata description = 'Creates an Azure Key Vault.' +param name string +param location string = resourceGroup().location +param tags object = {} + +param principalId string = '' + +resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' = { + name: name + location: location + tags: tags + properties: { + tenantId: subscription().tenantId + sku: { family: 'A', name: 'standard' } + accessPolicies: !empty(principalId) ? [ + { + objectId: principalId + permissions: { secrets: [ 'get', 'list' ] } + tenantId: subscription().tenantId + } + ] : [] + } +} + +output endpoint string = keyVault.properties.vaultUri +output name string = keyVault.name diff --git a/Environments/AKS-Store-Demo/core/security/registry-access.bicep b/Environments/AKS-Store-Demo/core/security/registry-access.bicep new file mode 100644 index 00000000..5335efab --- /dev/null +++ b/Environments/AKS-Store-Demo/core/security/registry-access.bicep @@ -0,0 +1,19 @@ +metadata description = 'Assigns ACR Pull permissions to access an Azure Container Registry.' +param containerRegistryName string +param principalId string + +var acrPullRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d') + +resource aksAcrPull 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: containerRegistry // Use when specifying a scope that is different than the deployment scope + name: guid(subscription().id, resourceGroup().id, principalId, acrPullRole) + properties: { + roleDefinitionId: acrPullRole + principalType: 'ServicePrincipal' + principalId: principalId + } +} + +resource containerRegistry 'Microsoft.ContainerRegistry/registries@2022-02-01-preview' existing = { + name: containerRegistryName +} diff --git a/Environments/AKS-Store-Demo/core/security/role.bicep b/Environments/AKS-Store-Demo/core/security/role.bicep new file mode 100644 index 00000000..0b30cfd3 --- /dev/null +++ b/Environments/AKS-Store-Demo/core/security/role.bicep @@ -0,0 +1,21 @@ +metadata description = 'Creates a role assignment for a service principal.' +param principalId string + +@allowed([ + 'Device' + 'ForeignGroup' + 'Group' + 'ServicePrincipal' + 'User' +]) +param principalType string = 'ServicePrincipal' +param roleDefinitionId string + +resource role 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(subscription().id, resourceGroup().id, principalId, roleDefinitionId) + properties: { + principalId: principalId + principalType: principalType + roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', roleDefinitionId) + } +} diff --git a/Environments/AKS-Store-Demo/main.bicep b/Environments/AKS-Store-Demo/main.bicep new file mode 100644 index 00000000..23842254 --- /dev/null +++ b/Environments/AKS-Store-Demo/main.bicep @@ -0,0 +1,227 @@ +@minLength(1) +@maxLength(64) +@description('Name of the the environment which is used to generate a short unique hash used in all resources.') +param environmentName string + +@minLength(1) +@description('Primary location for all resources') +param location string = resourceGroup().location + +// Optional parameters to override the default azd resource naming conventions. Update the main.parameters.json file to provide values. e.g.,: +// "resourceGroupName": { +// "value": "myGroupName" +// } +param k8s_namespace string = 'default' +param openAiServiceName string = '' +param openAiModelName string = 'gpt-35-turbo' +param identityName string = '' +param kubernetesName string = '' +param keyVaultName string = '' +param servicebusName string = '' +param logAnalyticsName string = '' +param monitorAccountName string = '' +param containerRegistryName string = '' + +@description('Deploy an Azure Container Registry or not') +param deployAcr bool = false + +@allowed([ + 'MongoDB' + 'GlobalDocumentDB' +]) +param cosmosdbAccountKind string = 'MongoDB' + +@description('Id of the user or app to assign application roles') +param principalId string = '' + +var abbrs = loadJsonContent('./abbreviations.json') +var resourceToken = toLower(uniqueString(subscription().id, environmentName, location)) +var tags = { 'azd-env-name': environmentName } + +// the node pool base configuration +var nodePoolBase = { + name: 'system' + count: 3 + vmSize: 'Standard_D4s_v4' +} + +// the openai deployments to create +var openAiDeployment = [ + { + name: openAiModelName + sku: { + name: 'Standard' + capacity: 30 + } + model: { + format: 'OpenAI' + name: openAiModelName + version: '0613' + } + } +] + +// create the openai resources +module openAi './core/ai/cognitiveservices.bicep' = { + name: 'openai' + params: { + name: !empty(openAiServiceName) ? openAiServiceName : '${abbrs.cognitiveServicesAccounts}${resourceToken}' + location: location + tags: tags + deployments: openAiDeployment + } +} + +// create the identity and assign roles +module identity './app/identity.bicep' = { + name: 'identity' + params: { + name: !empty(identityName) ? identityName : '${abbrs.managedIdentityUserAssignedIdentities}${resourceToken}' + location: location + tags: tags + principalId: principalId + AZURE_AKS_NAMESPACE: k8s_namespace + clusterName: kubernetes.outputs.clusterName + } +} + +// create the kunbernetes cluster +module kubernetes './app/aks-managed-cluster.bicep' = { + name: 'kubernetes' + params: { + name: !empty(kubernetesName) ? kubernetesName : '${abbrs.containerServiceManagedClusters}${resourceToken}' + location: location + tags: tags + networkPlugin: 'kubenet' + systemPoolConfig: union( + { name: 'npsystem', mode: 'System' }, + nodePoolBase + ) + dnsPrefix: !empty(kubernetesName) ? kubernetesName : '${abbrs.containerServiceManagedClusters}${resourceToken}' + } +} + +// store secrets in a keyvault +module keyVault './core/security/keyvault.bicep' = { + name: 'keyvault' + params: { + name: !empty(keyVaultName) ? keyVaultName : '${abbrs.keyVaultVaults}${resourceToken}' + location: location + tags: tags + principalId: principalId + } +} + +// create the cosmosdb +module cosmos './app/db.bicep' = { + name: 'cosmos' + params: { + resourceToken: resourceToken + location: location + tags: tags + kind: cosmosdbAccountKind + keyVaultName: keyVault.outputs.name + } +} + + +// create the service bus +module serviceBus './app/servicebus.bicep' = { + name: 'servicebus' + params: { + name: !empty(servicebusName) ? servicebusName : '${abbrs.serviceBusNamespaces}${resourceToken}' + location: location + tags: tags + keyVaultName: keyVault.outputs.name + } +} + +// get keys from the openAi and cosmosdb +module getKeys './app/get-keys.bicep' = { + name: 'get-keys' + params:{ + keyVaultName: keyVault.outputs.name + openAiName: openAi.outputs.name + cosmosAccountName: cosmos.outputs.name + } +} + +// create the monitor workspace +module monitor './app/monitor.bicep' = { + name: 'monitor' + params: { + name: !empty(monitorAccountName) ? monitorAccountName : 'amon-${resourceToken}' + location: location + tags: tags + } +} + +// create the log analytics workspace +module logAnalytics './core/monitor/loganalytics.bicep' = { + name: 'log-analytics' + params: { + name: !empty(logAnalyticsName) ? logAnalyticsName : '${abbrs.operationalInsightsWorkspaces}${resourceToken}' + location: location + tags: tags + } +} + +// create observability +module observability './app/observability.bicep' = { + name: 'observability' + params: { + name: 'amg-${resourceToken}' + principalId: principalId + clusterId: kubernetes.outputs.clusterId + clusterName: kubernetes.outputs.clusterName + logAnalyticsName: logAnalytics.outputs.name + logAnalyticsId: logAnalytics.outputs.id + monitorName: monitor.outputs.name + monitorId: monitor.outputs.id + location: location + tags: tags + } +} + +// create the container if the deployAcr is true +module containerRegistry './core/host/container-registry.bicep' = if(deployAcr) { + name: 'container-registry' + params: { + name: !empty(containerRegistryName) ? containerRegistryName : '${abbrs.containerRegistryRegistries}${resourceToken}' + location: location + tags: tags + sku: { + name: 'Premium' + } + } +} + +// acr pull role assignment +module acrPullRoleAssignment './core/security/registry-access.bicep' = if(deployAcr) { + name: 'acr-pull-role-assignment' + params: { + containerRegistryName: deployAcr ? containerRegistry.outputs.name : '' + principalId: kubernetes.outputs.clusterIdentity.objectId + } +} + +// outputs data +output AZURE_RESOURCEGROUP_NAME string = resourceGroup().name +output AZURE_AKS_CLUSTER_NAME string = kubernetes.outputs.clusterName +output AZURE_OPENAI_MODEL_NAME string = openAiModelName +output AZURE_OPENAI_ENDPOINT string = openAi.outputs.endpoint +output AZURE_IDENTITY_CLIENT_ID string = identity.outputs.clientId +output AZURE_SERVICE_BUS_HOST string = '${serviceBus.outputs.serviceBusNamespaceName}.servicebus.windows.net' +output AZURE_SERVICE_BUS_URI string = 'amqps://${serviceBus.outputs.serviceBusNamespaceName}.servicebus.windows.net' +output AZURE_SERVICE_BUS_LISTENER_NAME string = serviceBus.outputs.serviceBusListenerName +output AZURE_SERVICE_BUS_LISTENER_KEY string = serviceBus.outputs.serviceBusListenerKey +output AZURE_SERVICE_BUS_SENDER_NAME string = serviceBus.outputs.serviceBusSenderName +output AZURE_SERVICE_BUS_SENDER_KEY string = serviceBus.outputs.serviceBusSenderKey +output AZURE_COSMOS_DATABASE_NAME string = cosmos.outputs.name +output AZURE_COSMOS_DATABASE_URI string = cosmos.outputs.endpoint +output AZURE_COSMOS_DATABASE_KEY string = getKeys.outputs.cosmosKey +output AZURE_AKS_NAMESPACE string = k8s_namespace +output AZURE_KEY_VAULT_NAME string = keyVault.outputs.name +output AZURE_DATABASE_API string = cosmosdbAccountKind == 'MongoDB' ? 'mongodb': 'cosmosdbsql' +output AZURE_REGISTRY_NAME string = deployAcr ? containerRegistry.outputs.name : '' +output AZURE_REGISTRY_URI string = deployAcr ? containerRegistry.outputs.loginServer : 'ghcr.io/azure-samples' diff --git a/Environments/AKS-Store-Demo/manifest.yaml b/Environments/AKS-Store-Demo/manifest.yaml new file mode 100644 index 00000000..d7cc143c --- /dev/null +++ b/Environments/AKS-Store-Demo/manifest.yaml @@ -0,0 +1,28 @@ +name: eShop +version: 1.0.0 +summary: eShop Reference Application +description: eShop Reference Application - "Northern Mountains" +runner: ARM +templatePath: azuredeploy.json + +parameters: +- id: "environmentName" + name: "Environment Name (e.g. test)" + description: "Name of the Environment" + type: "string" + required: true + +- id: "location" + name: "Region (e.g. eastus)" + description: "Region" + type: "string" + required: false + default: "eastus" + +- id: "principalId" + name: "Principal Id (e.g. )" + description: "Id of the user or app to assign application roles" + type: "string" + required: true + +