diff --git a/.pipelines/e2e.yml b/.pipelines/e2e.yml index 1fb73a7c201..d0c3017b27c 100644 --- a/.pipelines/e2e.yml +++ b/.pipelines/e2e.yml @@ -40,13 +40,14 @@ jobs: - template: ./templates/template-push-images-to-acr.yml parameters: rpImageACR: $(RP_IMAGE_ACR) + acrCredentialsJSON: $(acr-credentials) - script: | make extract-aro-docker displayName: Extract ARO binaries from build - script: | az account set -s $AZURE_SUBSCRIPTION_ID - SECRET_SA_ACCOUNT_NAME=e2earosecrets make secrets + SECRET_SA_ACCOUNT_NAME=$(SECRET_SA_ACCOUNT_NAME) make secrets . secrets/env export CI=true diff --git a/.pipelines/templates/template-push-images-to-acr.yml b/.pipelines/templates/template-push-images-to-acr.yml index d9f8db03bb1..423f6b6ff7a 100644 --- a/.pipelines/templates/template-push-images-to-acr.yml +++ b/.pipelines/templates/template-push-images-to-acr.yml @@ -1,5 +1,6 @@ parameters: rpImageACR: "" + acrCredentialsJSON: "" steps: - script: | set -e @@ -7,7 +8,14 @@ steps: export RP_IMAGE_ACR=${{ parameters.rpImageACR }} - az acr login --name "$RP_IMAGE_ACR" + if [ -z ${{ parameters.acrCredentialsJSON }} ]; then + az acr login --name "$RP_IMAGE_ACR" + else + base64 -d >acr-credentials.json <<<${{ parameters.acrCredentialsJSON }} + az acr login --name "$RP_IMAGE_ACR" -u "$(jq -r .username < acr-credentials.json)" -p "$(jq -r .password < acr-credentials.json)" + rm -f acr-credentials.json + fi + # azure checkouts commit, so removing master reference when publishing image export BRANCH=$(Build.SourceBranchName) make publish-image-e2e diff --git a/docs/prepare-a-shared-rp-development-environment.md b/docs/prepare-a-shared-rp-development-environment.md index 93153e1119f..91efdf4b36c 100644 --- a/docs/prepare-a-shared-rp-development-environment.md +++ b/docs/prepare-a-shared-rp-development-environment.md @@ -11,13 +11,26 @@ locations. Azure subscription, as well as the ability to create and configure AAD applications. +1. Set the az account + ```bash + az account set -n "" + ``` + +1. You will need a resource group for global infrastructure + ```bash + GLOBAL_RESOURCEGROUP=global-infra + az group create -n $GLOBAL_RESOURCEGROUP --location eastus + ``` + 1. You will need a publicly resolvable DNS Zone resource in your Azure subscription. Set PARENT_DOMAIN_NAME and PARENT_DOMAIN_RESOURCEGROUP to the name and resource group of the DNS Zone resource: ```bash - PARENT_DOMAIN_NAME=osadev.cloud - PARENT_DOMAIN_RESOURCEGROUP=dns + PARENT_DOMAIN_NAME= + PARENT_DOMAIN_RESOURCEGROUP=global-infra + + az network dns zone create --name $PARENT_DOMAIN_NAME -g $PARENT_DOMAIN_RESOURCEGROUP ``` 1. You will need a storage account in your Azure subscription in which to store @@ -27,7 +40,9 @@ Data Reader` or `Storage Blob Data Contributor` role on the storage account. Set SECRET_SA_ACCOUNT_NAME to the name of the storage account: ```bash - SECRET_SA_ACCOUNT_NAME=e2earosecrets + export SECRET_SA_ACCOUNT_NAME= + ./hack/devtools/deploy-shared-env-storage.sh + ``` 1. You will need an AAD object (this could be your AAD user, or an AAD group of @@ -35,7 +50,7 @@ Data Reader` or `Storage Blob Data Contributor` role on the storage account. development environment key vault(s). Set ADMIN_OBJECT_ID to the object ID. ```bash - ADMIN_OBJECT_ID="$(az ad group show -g 'aro-engineering' --query id -o tsv)" + ADMIN_OBJECT_ID="$(az ad group show -g '' --query id -o tsv)" ``` 1. You will need the ARO RP-specific pull secret (ask one of the @@ -75,6 +90,11 @@ Data Reader` or `Storage Blob Data Contributor` role on the storage account. ## AAD applications +1. Set a prefix variable used for naming apps/sp +```bash +# for PR E2E Environment +PREFIX=aro-v4-e2e +``` 1. Create an AAD application which will fake up the ARM layer: This application requires client certificate authentication to be enabled. A @@ -89,7 +109,7 @@ Data Reader` or `Storage Blob Data Contributor` role on the storage account. ```bash > __NOTE:__: for macos change the -w0 option for base64 to -b0 AZURE_ARM_CLIENT_ID="$(az ad app create \ - --display-name aro-v4-arm-shared \ + --display-name ${PREFIX}-arm-shared \ --query appId \ -o tsv)" az ad app credential reset \ @@ -118,7 +138,7 @@ Data Reader` or `Storage Blob Data Contributor` role on the storage account. ```bash > __NOTE:__: for macos change the -w0 option for base64 to -b0 AZURE_FP_CLIENT_ID="$(az ad app create \ - --display-name aro-v4-fp-shared \ + --display-name ${PREFIX}-fp-shared \ --query appId \ -o tsv)" az ad app credential reset \ @@ -137,13 +157,7 @@ Data Reader` or `Storage Blob Data Contributor` role on the storage account. ```bash AZURE_RP_CLIENT_SECRET="$(uuidgen)" - AZURE_RP_CLIENT_ID="$(az ad app create \ - --display-name aro-v4-rp-shared \ - --end-date '2299-12-31T11:59:59+00:00' \ - --key-type password \ - --password "$AZURE_RP_CLIENT_SECRET" \ - --query appId \ - -o tsv)" + AZURE_RP_CLIENT_ID="$(az ad app create --display-name ${PREFIX}-rp-shared --end-date '2299-12-31T11:59:59+00:00' --key-type Password --key-value "$AZURE_RP_CLIENT_SECRET" --query appId -o tsv)" az ad sp create --id "$AZURE_RP_CLIENT_ID" >/dev/null ``` @@ -158,7 +172,7 @@ Data Reader` or `Storage Blob Data Contributor` role on the storage account. ```bash AZURE_GATEWAY_CLIENT_SECRET="$(uuidgen)" AZURE_GATEWAY_CLIENT_ID="$(az ad app create \ - --display-name aro-v4-gateway-shared \ + --display-name ${PREFIX}-gateway-shared \ --end-date '2299-12-31T11:59:59+00:00' \ --key-type password \ --password "$AZURE_GATEWAY_CLIENT_SECRET" \ @@ -172,7 +186,7 @@ Data Reader` or `Storage Blob Data Contributor` role on the storage account. ```bash AZURE_CLIENT_SECRET="$(uuidgen)" AZURE_CLIENT_ID="$(az ad app create \ - --display-name aro-v4-tooling-shared \ + --display-name ${PREFIX}-tooling-shared \ --end-date '2299-12-31T11:59:59+00:00' \ --key-type password \ --password "$AZURE_CLIENT_SECRET" \ @@ -228,10 +242,17 @@ Data Reader` or `Storage Blob Data Contributor` role on the storage account. ```bash > __NOTE:__: for macos change the -w0 option for base64 to -b0 AZURE_PORTAL_CLIENT_ID="$(az ad app create \ - --display-name aro-v4-portal-shared \ - --reply-urls "https://localhost:8444/callback" \ + --display-name ${PREFIX}-portal-shared \ --query appId \ -o tsv)" + + OBJ_ID="$(az ad app show --id $AZURE_PORTAL_CLIENT_ID --query id -o tsv)" + + az rest --method PATCH \ + --uri "https://graph.microsoft.com/v1.0/applications/$OBJ_ID" \ + --headers 'Content-Type=application/json' \ + --body '{"web":{"redirectUris":["https://locahlost:8444/callback"]}}' + az ad app credential reset \ --id "$AZURE_PORTAL_CLIENT_ID" \ --cert "$(base64 -w0 /dev/null @@ -302,6 +323,14 @@ Data Reader` or `Storage Blob Data Contributor` role on the storage account. mv dev-client.* secrets ``` +1. Create the CA key/certificate. A suitable key/certificate file can be + generated using the following helper utility: + + ```bash + go run ./hack/genkey cluster-mdsd + mv cluster-mdsd.* secrets + ``` + ## Certificate Rotation This section documents the steps taken to rotate certificates in dev and INT subscriptions @@ -348,7 +377,7 @@ az ad app credential reset \ 5. The RP makes API calls to kubernetes cluster via a proxy VMSS agent. For the agent to get the updated certificates, this vm needs to be deleted & redeployed. Proxy VM is currently deployed by the `deploy_env_dev` function in `deploy-shared-env.sh`. It makes use of `env-development.json` -6. Run `[rharosecretsdev|e2earosecrets] make secrets-update` to upload it to your +6. Run `[rharosecretsdev|e2earosecrets|e2earoclassicsecrets] make secrets-update` to upload it to your storage account so other people on your team can access it via `make secrets` # Environment file @@ -357,14 +386,14 @@ az ad app credential reset \ The resource group location will be appended to the prefix to make the resource group name. If a v4-prefixed environment exists in the subscription already, use a unique prefix. ```bash - RESOURCEGROUP_PREFIX=v4 + RESOURCEGROUP_PREFIX= ``` 1. Choose the proxy domain name label. This final proxy hostname will be of the form `vm0.$PROXY_DOMAIN_NAME_LABEL.$LOCATION.cloudapp.azure.com`. ```bash - PROXY_DOMAIN_NAME_LABEL=aroproxy + PROXY_DOMAIN_NAME_LABEL= ``` 1. Create the secrets/env file: @@ -422,7 +451,11 @@ each of the bash functions below. - LOCATION: Location of the shared RP development environment (default: `eastus`). - +1. Create AzSecPack managed Identity https://msazure.visualstudio.com/ASMDocs/_wiki/wikis/ASMDocs.wiki/234249/AzSecPack-AutoConfig-UserAssigned-Managed-Identity (required for `deploy_env_dev`) +1. Enable EncryptionAtHost for subscription. + ```bash + az feature register --namespace Microsoft.Compute --name EncryptionAtHost + ``` 1. Create the resource group and deploy the RP resources: ```bash @@ -433,6 +466,8 @@ each of the bash functions below. deploy_rp_dev_predeploy # Deploy the infrastructure resources such as Cosmos, KV, Vnet... deploy_rp_dev + # Deploy RP MSI for aks/hive + deploy_rp_managed_identity # Deploy the proxy and VPN deploy_env_dev # Deploy AKS resources for Hive @@ -520,6 +555,18 @@ Vault Name: "$KEYVAULT_PREFIX-svc" Certificate: cluster-mdsd Development value: secrets/cluster-logging-int.pem ``` + > __NOTE:__: in the new tenant OneCert is not available, therefore firstparty and cluster-mdsd are self signed. + ```bash + az keyvault certificate import \ + --vault-name "$KEYVAULT_PREFIX-svc" \ + --name rp-firstparty \ + --file secrets/firstparty.pem + + az keyvault certificate import \ + --vault-name "$KEYVAULT_PREFIX-svc" \ + --name cluster-mdsd \ + --file secrets/cluster-mdsd.pem + ``` 1. Create nameserver records in the parent DNS zone: @@ -533,6 +580,40 @@ Development value: secrets/cluster-logging-int.pem vpn_configuration ``` +## PR E2E Only - Create the global keyvault, ADO Library Variable Group + +1. Create E2E global keyvault + ```bash + AZURE_TENANT_ID=$(az account show --query tenantId -o tsv) + ARO_E2E_GLOBAL_VAULT_NAME= + + deploy_aro_e2e_global_keyvault + ``` +1. Upload Keyvault Secrets and Certificates +1. Give List/Get permissions to Azure DevOps Connection SPN +1. Set up Library Variable group in ADO and connect it to keyvault + +## PR E2E Only - Setup ACR Credentials +Due to cross tenant ACR access, token credentials must be generated for arointsvc +1. Login to MSIT tenant and navigate to arointsvc +1. Under "Repository permissions -> Tokens" add a new token and generate a password +1. Add username and password to json file formatted like below and convert it to base64 +``` +{ + "username": "", + "password": "" +} +``` +1. convert to base 64, copy the output and add it to aro-e2e-global keyvault +``` +cat .json | base64 -w0 +``` +1. Add the secret to the Libary variable group that is connected to the global keyvault + +## PR E2E Only - Add keyvault permissions to aro-v4-e2e-devops-spn +- assign 'Keyvault Secrets User' to aro-v4-e2e-devops-spn + + ## Append Resource Group to Subscription Cleaner DenyList - We have subscription pruning that takes place routinely and need to add our resource group for the shared rp environment to the `denylist` of the cleaner: diff --git a/hack/devtools/deploy-shared-env.sh b/hack/devtools/deploy-shared-env.sh index 4835a6cc231..14b7b4b0955 100644 --- a/hack/devtools/deploy-shared-env.sh +++ b/hack/devtools/deploy-shared-env.sh @@ -60,6 +60,14 @@ deploy_oic_dev() { "storageAccountDomain=$(echo ${RESOURCEGROUP//-})" >/dev/null } +deploy_rp_managed_identity() { + echo "########## Deploying RP Managed Identity (for hive/aks) in RG $RESOURCEGROUP ##########" + az deployment group create \ + -g "$RESOURCEGROUP" \ + -n rp-managed-identity \ + --template-file pkg/deploy/assets/rp-production-managed-identity.json +} + deploy_aks_dev() { echo "########## Deploying aks-development in RG $RESOURCEGROUP ##########" az deployment group create \ @@ -240,6 +248,35 @@ clean_env() { done } +deploy_e2e_secret_storage() { + az deployment group create \ + --name e2esecretstorage \ + --resource-group global-infra \ + --parameters storageAccounts_e2earosecrets_name=$SECRET_SA_ACCOUNT_NAME \ + --template-file pkg/deploy/assets/e2e-secret-storage.json +} + +deploy_aro_spn_keyvault() { + az deployment group create \ + --name aroe2eprincipals \ + --resource-group global-infra \ + --parameters \ + "vaults_aro_e2e_principals_name=$VAULTS_ARO_E2E_PRINCIPALS_NAME" \ + "tenant_id=$AZURE_TENANT_ID" \ + --template-file pkg/deploy/assets/e2e-aro-spn-keyvault.json +} + +deploy_aro_e2e_global_keyvault() { + az deployment group create \ + --name aroe2eprincipals \ + --resource-group global-infra \ + --parameters \ + "vault_name=$ARO_E2E_GLOBAL_VAULT_NAME" \ + "tenant_id=$AZURE_TENANT_ID" \ + --template-file pkg/deploy/assets/e2e-global-keyvault.json + +} + echo "##########################################" echo "##### ARO V4 Dev Env helper sourced ######" echo "##########################################" diff --git a/hack/e2e/run-rp-and-e2e.sh b/hack/e2e/run-rp-and-e2e.sh index 721f076ac8a..aee67f90b59 100755 --- a/hack/e2e/run-rp-and-e2e.sh +++ b/hack/e2e/run-rp-and-e2e.sh @@ -217,14 +217,13 @@ delete_e2e_cluster() { get_cluster_sp() { echo "########## Downloading SP secrets ##########" - - az keyvault secret download --vault-name=aro-e2e-principals \ + az keyvault secret download --vault-name=$CSP_VAULT_NAME \ --name=aro-v4-e2e-devops-spn-1-app-id \ --file=secrets/app-id - az keyvault secret download --vault-name=aro-e2e-principals \ + az keyvault secret download --vault-name=$CSP_VAULT_NAME \ --name=aro-v4-e2e-devops-spn-1-sp-id \ --file=secrets/sp-id - az keyvault secret download --vault-name=aro-e2e-principals \ + az keyvault secret download --vault-name=$CSP_VAULT_NAME \ --name=aro-v4-e2e-devops-spn-1-secret-value \ --file=secrets/secret-value diff --git a/hack/hive-generate-config.sh b/hack/hive-generate-config.sh index caa64f1225b..4e4fd115225 100755 --- a/hack/hive-generate-config.sh +++ b/hack/hive-generate-config.sh @@ -1,7 +1,7 @@ #!/bin/bash # This is the commit sha that the image was built from and ensures we use the correct configs for the release -HIVE_IMAGE_COMMIT_HASH="5fbe0d158b" +HIVE_IMAGE_COMMIT_HASH="d7ead609f4" # For now we'll use the quay hive image, but this will change to an ACR once the quay.io -> ACR mirroring is setup # Note: semi-scientific way to get the latest image: `podman search --list-tags --limit 10000 quay.io/app-sre/hive | tail -n1` diff --git a/pkg/deploy/assets/e2e-aro-spn-keyvault.json b/pkg/deploy/assets/e2e-aro-spn-keyvault.json new file mode 100644 index 00000000000..37391cfaa08 --- /dev/null +++ b/pkg/deploy/assets/e2e-aro-spn-keyvault.json @@ -0,0 +1,39 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "vaults_aro_e2e_principals_name": { + "defaultValue": "aro-classic-e2e-principals", + "type": "String" + }, + "tenant_id":{ + "defaultValue": "93b21e64-4824-439a-b893-46c9b2a51082", + "type": "String" + } + }, + "variables": {}, + "resources": [ + { + "type": "Microsoft.KeyVault/vaults", + "apiVersion": "2023-07-01", + "name": "[parameters('vaults_aro_e2e_principals_name')]", + "location": "eastus", + "properties": { + "sku": { + "family": "A", + "name": "Standard" + }, + "tenantId": "[parameters('tenant_id')]", + "accessPolicies": [], + "enabledForDeployment": false, + "enabledForDiskEncryption": false, + "enabledForTemplateDeployment": false, + "enableSoftDelete": true, + "softDeleteRetentionInDays": 90, + "enableRbacAuthorization": true, + "vaultUri": "[concat('https://', parameters('vaults_aro_e2e_principals_name'), '.vault.azure.net/')]", + "publicNetworkAccess": "Enabled" + } + } + ] +} diff --git a/pkg/deploy/assets/e2e-global-keyvault.json b/pkg/deploy/assets/e2e-global-keyvault.json new file mode 100644 index 00000000000..7f556e5cb3f --- /dev/null +++ b/pkg/deploy/assets/e2e-global-keyvault.json @@ -0,0 +1,39 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "vault_name": { + "defaultValue": "", + "type": "String" + }, + "tenant_id": { + "defaultValue": "", + "type": "String" + } + }, + "variables": {}, + "resources": [ + { + "type": "Microsoft.KeyVault/vaults", + "apiVersion": "2023-07-01", + "name": "[parameters('vault_name')]", + "location": "eastus", + "properties": { + "sku": { + "family": "A", + "name": "standard" + }, + "tenantId": "[parameters('tenant_id')]", + "accessPolicies": [], + "enabledForDeployment": false, + "enabledForDiskEncryption": false, + "enabledForTemplateDeployment": false, + "enableSoftDelete": true, + "softDeleteRetentionInDays": 90, + "enableRbacAuthorization": false, + "vaultUri": "[concat('https://', parameters('vault_name'), '.vault.azure.net/')]", + "publicNetworkAccess": "Enabled" + } + } + ] +} \ No newline at end of file diff --git a/pkg/deploy/assets/e2e-secret-storage.json b/pkg/deploy/assets/e2e-secret-storage.json new file mode 100644 index 00000000000..f880ef33036 --- /dev/null +++ b/pkg/deploy/assets/e2e-secret-storage.json @@ -0,0 +1,155 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "storageAccounts_e2earosecrets_name": { + "defaultValue": "e2earoclassicsecrets", + "type": "String" + } + }, + "variables": {}, + "resources": [ + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2023-04-01", + "name": "[parameters('storageAccounts_e2earosecrets_name')]", + "location": "eastus", + "sku": { + "name": "Standard_RAGRS", + "tier": "Standard" + }, + "kind": "StorageV2", + "properties": { + "dnsEndpointType": "Standard", + "defaultToOAuthAuthentication": false, + "publicNetworkAccess": "Enabled", + "allowCrossTenantReplication": true, + "minimumTlsVersion": "TLS1_2", + "allowBlobPublicAccess": true, + "allowSharedKeyAccess": true, + "networkAcls": { + "bypass": "AzureServices", + "virtualNetworkRules": [], + "ipRules": [], + "defaultAction": "Allow" + }, + "supportsHttpsTrafficOnly": true, + "encryption": { + "requireInfrastructureEncryption": false, + "services": { + "file": { + "keyType": "Account", + "enabled": true + }, + "blob": { + "keyType": "Account", + "enabled": true + } + }, + "keySource": "Microsoft.Storage" + }, + "accessTier": "Hot" + } + }, + { + "type": "Microsoft.Storage/storageAccounts/blobServices", + "apiVersion": "2023-04-01", + "name": "[concat(parameters('storageAccounts_e2earosecrets_name'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccounts_e2earosecrets_name'))]" + ], + "sku": { + "name": "Standard_RAGRS", + "tier": "Standard" + }, + "properties": { + "changeFeed": { + "enabled": false + }, + "restorePolicy": { + "enabled": false + }, + "containerDeleteRetentionPolicy": { + "enabled": true, + "days": 7 + }, + "cors": { + "corsRules": [] + }, + "deleteRetentionPolicy": { + "allowPermanentDelete": false, + "enabled": true, + "days": 7 + }, + "isVersioningEnabled": true + } + }, + { + "type": "Microsoft.Storage/storageAccounts/fileServices", + "apiVersion": "2023-04-01", + "name": "[concat(parameters('storageAccounts_e2earosecrets_name'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccounts_e2earosecrets_name'))]" + ], + "sku": { + "name": "Standard_RAGRS", + "tier": "Standard" + }, + "properties": { + "protocolSettings": { + "smb": {} + }, + "cors": { + "corsRules": [] + }, + "shareDeleteRetentionPolicy": { + "enabled": true, + "days": 7 + } + } + }, + { + "type": "Microsoft.Storage/storageAccounts/queueServices", + "apiVersion": "2023-04-01", + "name": "[concat(parameters('storageAccounts_e2earosecrets_name'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccounts_e2earosecrets_name'))]" + ], + "properties": { + "cors": { + "corsRules": [] + } + } + }, + { + "type": "Microsoft.Storage/storageAccounts/tableServices", + "apiVersion": "2023-04-01", + "name": "[concat(parameters('storageAccounts_e2earosecrets_name'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccounts_e2earosecrets_name'))]" + ], + "properties": { + "cors": { + "corsRules": [] + } + } + }, + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(parameters('storageAccounts_e2earosecrets_name'), '/default/secrets')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', parameters('storageAccounts_e2earosecrets_name'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccounts_e2earosecrets_name'))]" + ], + "properties": { + "immutableStorageWithVersioning": { + "enabled": false + }, + "defaultEncryptionScope": "$account-encryption-key", + "denyEncryptionScopeOverride": false, + "publicAccess": "None" + } + } + ] +} \ No newline at end of file diff --git a/pkg/deploy/assets/env-development.json b/pkg/deploy/assets/env-development.json index dfa6575961e..b1b62615d26 100644 --- a/pkg/deploy/assets/env-development.json +++ b/pkg/deploy/assets/env-development.json @@ -407,7 +407,7 @@ "identity": { "type": "UserAssigned", "userAssignedIdentities": { - "[resourceId('AzSecPackAutoConfigRG', 'Microsoft.ManagedIdentity/userAssignedIdentities', concat('AzSecPackAutoConfigUA-', resourceGroup().location))]": {} + "[resourceId('AzSecPackAutoConfigRG', 'Microsoft.ManagedIdentity/userAssignedIdentities', 'AzSecPackAutoConfigUA-eastus')]": {} } }, "name": "dev-proxy-vmss", diff --git a/pkg/deploy/assets/rp-development.json b/pkg/deploy/assets/rp-development.json index b488666b969..05a860407b0 100644 --- a/pkg/deploy/assets/rp-development.json +++ b/pkg/deploy/assets/rp-development.json @@ -82,6 +82,20 @@ }, "type": "Microsoft.DocumentDB/databaseAccounts" }, + { + "name": "[concat(parameters('databaseAccountName'), '/', guid(resourceId('Microsoft.DocumentDB/databaseAccounts', parameters('databaseAccountName')), parameters('rpServicePrincipalId'), 'DocumentDB Data Contributor'))]", + "type": "Microsoft.DocumentDB/databaseAccounts/sqlRoleAssignments", + "properties": { + "scope": "[resourceId('Microsoft.DocumentDB/databaseAccounts/', parameters('databaseAccountName'))]", + "roleDefinitionId": "[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlRoleDefinitions', parameters('databaseAccountName'), '00000000-0000-0000-0000-000000000002')]", + "principalId": "[parameters('rpServicePrincipalId')]", + "principalType": "ServicePrincipal" + }, + "apiVersion": "2023-04-15", + "dependsOn": [ + "[resourceId('Microsoft.DocumentDB/databaseAccounts', parameters('databaseAccountName'))]" + ] + }, { "name": "[guid(resourceGroup().id, parameters('rpServicePrincipalId'), 'RP / Reader')]", "type": "Microsoft.Authorization/roleAssignments", diff --git a/pkg/deploy/generator/resources_dev.go b/pkg/deploy/generator/resources_dev.go index 7f8be1347e8..91b4895ef76 100644 --- a/pkg/deploy/generator/resources_dev.go +++ b/pkg/deploy/generator/resources_dev.go @@ -117,7 +117,7 @@ func (g *generator) devProxyVMSS() *arm.Resource { Identity: &mgmtcompute.VirtualMachineScaleSetIdentity{ Type: mgmtcompute.ResourceIdentityTypeUserAssigned, UserAssignedIdentities: map[string]*mgmtcompute.VirtualMachineScaleSetIdentityUserAssignedIdentitiesValue{ - "[resourceId('AzSecPackAutoConfigRG', 'Microsoft.ManagedIdentity/userAssignedIdentities', concat('AzSecPackAutoConfigUA-', resourceGroup().location))]": {}, + "[resourceId('AzSecPackAutoConfigRG', 'Microsoft.ManagedIdentity/userAssignedIdentities', 'AzSecPackAutoConfigUA-eastus')]": {}, }, }, VirtualMachineScaleSetProperties: &mgmtcompute.VirtualMachineScaleSetProperties{ diff --git a/pkg/deploy/generator/resources_rp.go b/pkg/deploy/generator/resources_rp.go index 4d62368cbd1..9b2a4f2ed83 100644 --- a/pkg/deploy/generator/resources_rp.go +++ b/pkg/deploy/generator/resources_rp.go @@ -875,28 +875,38 @@ func (g *generator) rpCosmosDB() []*arm.Resource { rs = append(rs, g.rpCosmosDBAlert(10, 90, 3, "rp-cosmosdb-alert", "PT5M", "PT1H")) rs = append(rs, g.CosmosDBDataContributorRoleAssignment("'ARO'", "rp")) rs = append(rs, g.CosmosDBDataContributorRoleAssignment("'ARO'", "gateway")) + } else { + rs = append(rs, g.CosmosDBDataContributorRoleAssignment("''", "rp")) } return rs } func (g *generator) CosmosDBDataContributorRoleAssignment(databaseName, component string) *arm.Resource { - return &arm.Resource{ + var scope string + if g.production { + scope = "[resourceId('Microsoft.DocumentDB/databaseAccounts/dbs', parameters('databaseAccountName'), " + databaseName + ")]" + } else { + scope = "[resourceId('Microsoft.DocumentDB/databaseAccounts/', parameters('databaseAccountName'))]" + } + + roleAssignment := &arm.Resource{ Resource: mgmtauthorization.RoleAssignment{ Name: to.StringPtr("[concat(parameters('databaseAccountName'), '/', guid(resourceId('Microsoft.DocumentDB/databaseAccounts', parameters('databaseAccountName')), parameters('" + component + "ServicePrincipalId'), 'DocumentDB Data Contributor'))]"), Type: to.StringPtr("Microsoft.DocumentDB/databaseAccounts/sqlRoleAssignments"), RoleAssignmentPropertiesWithScope: &mgmtauthorization.RoleAssignmentPropertiesWithScope{ - Scope: to.StringPtr("[resourceId('Microsoft.DocumentDB/databaseAccounts/dbs', parameters('databaseAccountName'), " + databaseName + ")]"), + Scope: &scope, RoleDefinitionID: to.StringPtr("[resourceId('Microsoft.DocumentDB/databaseAccounts/sqlRoleDefinitions', parameters('databaseAccountName'), '" + rbac.RoleDocumentDBDataContributor + "')]"), PrincipalID: to.StringPtr("[parameters('" + component + "ServicePrincipalId')]"), PrincipalType: mgmtauthorization.ServicePrincipal, }, }, - APIVersion: azureclient.APIVersion("Microsoft.DocumentDB"), DependsOn: []string{ "[resourceId('Microsoft.DocumentDB/databaseAccounts', parameters('databaseAccountName'))]", }, + APIVersion: azureclient.APIVersion("Microsoft.DocumentDB"), } + return roleAssignment } func (g *generator) database(databaseName string, addDependsOn bool) []*arm.Resource { diff --git a/test/e2e/operator.go b/test/e2e/operator.go index 33be4deecfa..b245476dd0a 100644 --- a/test/e2e/operator.go +++ b/test/e2e/operator.go @@ -124,6 +124,9 @@ var _ = Describe("ARO Operator - Internet checking", func() { var _ = Describe("ARO Operator - Geneva Logging", func() { It("must be repaired if DaemonSet deleted", func(ctx context.Context) { + if _env.IsLocalDevelopmentMode() { + Skip("skipping tests in development environment") + } mdsdIsReady := func(g Gomega, ctx context.Context) { done, err := ready.CheckDaemonSetIsReady(ctx, clients.Kubernetes.AppsV1().DaemonSets("openshift-azure-logging"), "mdsd")()