diff --git a/.github/workflows/config/container-environment-config.yaml b/.github/workflows/config/container-environment-config.yaml deleted file mode 100644 index 3006337..0000000 --- a/.github/workflows/config/container-environment-config.yaml +++ /dev/null @@ -1,2 +0,0 @@ -properties: - managedEnvironmentId: /subscriptions/c57c2a02-297d-4f68-979b-1960df722627/resourceGroups/onlinestore-shared-rg/providers/Microsoft.App/managedEnvironments/onlinestore-containerapps diff --git a/.github/workflows/instance-deploy-prod.yml b/.github/workflows/instance-deploy-prod.yml index e0c7c26..f4cd26c 100644 --- a/.github/workflows/instance-deploy-prod.yml +++ b/.github/workflows/instance-deploy-prod.yml @@ -80,9 +80,6 @@ jobs: run: | mkdir terraform-outputs terraform -chdir=instance output -raw resource_group_name > terraform-outputs/resource_group_name.txt - terraform -chdir=instance output -raw container_app_api_fqdn > terraform-outputs/container_app_api_fqdn.txt - terraform -chdir=instance output -raw container_app_website_fqdn > terraform-outputs/container_app_website_fqdn.txt - terraform -chdir=instance show -json | jq -r '.values.outputs.container_app_monitoring_fqdn.value // ""' > container_app_monitoring_fqdn.txt - name: Upload terraform outputs for deploy job uses: actions/upload-artifact@v3 with: @@ -105,9 +102,6 @@ jobs: shell: bash run: | echo "resource_group_name=$(cat resource_group_name.txt)" >> $GITHUB_ENV - echo "container_app_api_fqdn=$(cat container_app_api_fqdn.txt)" >> $GITHUB_ENV - echo "container_app_website_fqdn=$(cat container_app_website_fqdn.txt)" >> $GITHUB_ENV - echo "container_app_monitoring_fqdn=$(cat container_app_monitoring_fqdn.txt)" >> $GITHUB_ENV - name: Login via Azure CLI uses: azure/login@v1 with: @@ -115,26 +109,24 @@ jobs: - name: Deploy api uses: azure/container-apps-deploy-action@v1 with: - yamlConfigPath: .github/workflows/config/container-environment-config.yaml acrName: onlinestorecontainerregistry acrUsername: ${{ secrets.ACR_USERNAME }} acrPassword: ${{ secrets.ACR_TOKEN }} - containerAppName: onlinestore-api + containerAppName: prod-onlinestore-api imageToDeploy: onlinestorecontainerregistry.azurecr.io/onlinestore-api:${{ github.sha }} location: 'East US' - resourceGroup: ${{ env.resource_group_name }} + resourceGroup: onlinestore-shared-rg targetPort: 8080 - environmentVariables: OTEL_EXPORTER_OTLP_ENDPOINT=http://onlinestore-monitoring:18889 + environmentVariables: OTEL_EXPORTER_OTLP_ENDPOINT=http://prod-onlinestore-monitoring:18889 - name: Deploy website uses: azure/container-apps-deploy-action@v1 with: - yamlConfigPath: .github/workflows/config/container-environment-config.yaml acrName: onlinestorecontainerregistry acrUsername: ${{ secrets.ACR_USERNAME }} acrPassword: ${{ secrets.ACR_TOKEN }} - containerAppName: onlinestore-website + containerAppName: prod-onlinestore-website imageToDeploy: onlinestorecontainerregistry.azurecr.io/onlinestore-website:${{ github.sha }} location: 'East US' - resourceGroup: ${{ env.resource_group_name }} + resourceGroup: onlinestore-shared-rg targetPort: 80 environmentVariables: "API__BASEPATH=https://api.rockpal.co.uk" diff --git a/.github/workflows/instance-deploy-test.yml b/.github/workflows/instance-deploy-test.yml index ca52a24..e099908 100644 --- a/.github/workflows/instance-deploy-test.yml +++ b/.github/workflows/instance-deploy-test.yml @@ -43,6 +43,10 @@ jobs: uses: hashicorp/setup-terraform@v2 with: terraform_wrapper: false + - name: Login via Azure CLI + uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} - name: Terraform Format id: fmt run: terraform -chdir=instance fmt @@ -60,7 +64,7 @@ jobs: TF_VAR_environment: ${{ github.head_ref }} TF_VAR_acr_username: ${{ secrets.ACR_USERNAME }} TF_VAR_acr_password: ${{ secrets.ACR_TOKEN }} - TF_VAR_website_dns_subdomain: ${{ github.head_ref }}-site + TF_VAR_website_dns_subdomain: ${{ github.head_ref }}-website TF_VAR_api_dns_subdomain: ${{ github.head_ref }}-api TF_VAR_monitoring_dns_subdomain: ${{ github.head_ref }}-monitoring - name: Terraform Apply @@ -71,17 +75,14 @@ jobs: TF_VAR_environment: ${{ github.head_ref }} TF_VAR_acr_username: ${{ secrets.ACR_USERNAME }} TF_VAR_acr_password: ${{ secrets.ACR_TOKEN }} - TF_VAR_website_dns_subdomain: ${{ github.head_ref }} - TF_VAR_api_dns_subdomain: api - TF_VAR_monitoring_dns_subdomain: monitoring + TF_VAR_website_dns_subdomain: ${{ github.head_ref }}-website + TF_VAR_api_dns_subdomain: ${{ github.head_ref }}-api + TF_VAR_monitoring_dns_subdomain: ${{ github.head_ref }}-monitoring - name: Save terraform outputs shell: bash run: | mkdir terraform-outputs terraform -chdir=instance output -raw resource_group_name > terraform-outputs/resource_group_name.txt - terraform -chdir=instance output -raw container_app_api_fqdn > terraform-outputs/container_app_api_fqdn.txt - terraform -chdir=instance output -raw container_app_website_fqdn > terraform-outputs/container_app_website_fqdn.txt - terraform -chdir=instance output -raw container_app_monitoring_fqdn > terraform-outputs/container_app_monitoring_fqdn.txt - name: Upload terraform outputs for deploy job uses: actions/upload-artifact@v3 with: @@ -104,9 +105,6 @@ jobs: shell: bash run: | echo "resource_group_name=$(cat resource_group_name.txt)" >> $GITHUB_ENV - echo "container_app_api_fqdn=$(cat container_app_api_fqdn.txt)" >> $GITHUB_ENV - echo "container_app_website_fqdn=$(cat container_app_website_fqdn.txt)" >> $GITHUB_ENV - echo "container_app_monitoring_fqdn=$(cat container_app_monitoring_fqdn.txt)" >> $GITHUB_ENV - name: Login via Azure CLI uses: azure/login@v1 with: @@ -117,22 +115,22 @@ jobs: acrName: onlinestorecontainerregistry acrUsername: ${{ secrets.ACR_USERNAME }} acrPassword: ${{ secrets.ACR_TOKEN }} - containerAppName: onlinestore-api + containerAppName: ${{ github.head_ref }}-onlinestore-api imageToDeploy: onlinestorecontainerregistry.azurecr.io/onlinestore-api:${{ github.sha }} location: 'East US' - resourceGroup: ${{ env.resource_group_name }} + resourceGroup: onlinestore-shared-rg targetPort: 8080 - environmentVariables: OTEL_EXPORTER_OTLP_ENDPOINT=http://onlinestore-monitoring:18889 + environmentVariables: OTEL_EXPORTER_OTLP_ENDPOINT=http://${{ github.head_ref }}-onlinestore-monitoring:18889 - name: Deploy website uses: azure/container-apps-deploy-action@v1 with: acrName: onlinestorecontainerregistry acrUsername: ${{ secrets.ACR_USERNAME }} acrPassword: ${{ secrets.ACR_TOKEN }} - containerAppName: onlinestore-website + containerAppName: ${{ github.head_ref }}-onlinestore-website imageToDeploy: onlinestorecontainerregistry.azurecr.io/onlinestore-website:${{ github.sha }} location: 'East US' - resourceGroup: ${{ env.resource_group_name }} + resourceGroup: onlinestore-shared-rg targetPort: 80 environmentVariables: "API__BASEPATH=https://${{ github.head_ref }}-api.rockpal.co.uk" - name: Find Comment @@ -150,7 +148,7 @@ jobs: edit-mode: replace body: | ### Test environment information - #### 🔗 [Company Website](https://${{ github.head_ref }}-site.rockpal.co.uk/) + #### 🔗 [Company Website](https://${{ github.head_ref }}-website.rockpal.co.uk/) #### 🔗 [Company API](https://${{ github.head_ref }}-api.rockpal.co.uk/swagger/) #### 🔗 [Monitoring Dashboard](https://${{ github.head_ref }}-monitoring.rockpal.co.uk/) diff --git a/.github/workflows/instance-destroy-test.yaml b/.github/workflows/instance-destroy-test.yml similarity index 91% rename from .github/workflows/instance-destroy-test.yaml rename to .github/workflows/instance-destroy-test.yml index 6c660d0..d8c04f0 100644 --- a/.github/workflows/instance-destroy-test.yaml +++ b/.github/workflows/instance-destroy-test.yml @@ -31,3 +31,5 @@ jobs: TF_VAR_acr_username: "not-used" TF_VAR_acr_password: "not-used" TF_VAR_website_dns_subdomain: "not-used" + TF_VAR_api_dns_subdomain: "not-used" + TF_VAR_monitoring_dns_subdomain: "not-used" diff --git a/terraform/instance/container_apps.tf b/terraform/instance/container_apps.tf index aec3ff8..f30baaa 100644 --- a/terraform/instance/container_apps.tf +++ b/terraform/instance/container_apps.tf @@ -4,14 +4,14 @@ data "azurerm_container_app_environment" "apps" { } resource "azurerm_container_app" "api" { - name = "${var.name}-api" + name = "${lower(var.environment)}-${var.name}-api" container_app_environment_id = data.azurerm_container_app_environment.apps.id - resource_group_name = azurerm_resource_group.instance.name + resource_group_name = data.azurerm_container_app_environment.apps.resource_group_name revision_mode = "Single" template { container { - name = "onlinestore-api" + name = "${lower(var.environment)}-onlinestore-api" image = "mcr.microsoft.com/azuredocs/containerapps-helloworld:latest" cpu = 0.25 memory = "0.5Gi" @@ -39,14 +39,14 @@ resource "azurerm_container_app" "api" { } resource "azurerm_container_app" "website" { - name = "${var.name}-website" + name = "${lower(var.environment)}-${var.name}-website" container_app_environment_id = data.azurerm_container_app_environment.apps.id - resource_group_name = azurerm_resource_group.instance.name + resource_group_name = data.azurerm_container_app_environment.apps.resource_group_name revision_mode = "Single" template { container { - name = "onlinestore-website" + name = "${lower(var.environment)}-onlinestore-website" image = "mcr.microsoft.com/azuredocs/containerapps-helloworld:latest" cpu = 0.25 memory = "0.5Gi" @@ -74,14 +74,14 @@ resource "azurerm_container_app" "website" { } resource "azurerm_container_app" "monitoring" { - name = "${var.name}-monitoring" + name = "${lower(var.environment)}-${var.name}-monitoring" container_app_environment_id = data.azurerm_container_app_environment.apps.id - resource_group_name = azurerm_resource_group.instance.name + resource_group_name = data.azurerm_container_app_environment.apps.resource_group_name revision_mode = "Single" template { container { - name = "aspire-dashboard" + name = "${lower(var.environment)}-aspire-dashboard" image = "onlinestorecontainerregistry.azurecr.io/dotnet/aspire-dashboard:8.0.0" cpu = 0.25 memory = "0.5Gi" diff --git a/terraform/instance/container_apps_bind_dns/main.tf b/terraform/instance/container_apps_bind_dns/main.tf new file mode 100644 index 0000000..2958914 --- /dev/null +++ b/terraform/instance/container_apps_bind_dns/main.tf @@ -0,0 +1,44 @@ +terraform {} + +resource "null_resource" "null" { + for_each = { for svc in var.services : svc.key => svc } + + lifecycle { + create_before_destroy = false + } + + triggers = { + ca_name = each.value.container_app_name + ca_rg_name = var.container_app_resource_group_name + ca_env_name = var.container_app_env_name + ca_env_rg_name = var.container_app_env_resource_group_name + custom_domain = each.value.custom_domain + } + + # provision a managed cert and apply it to the container app + provisioner "local-exec" { + when = create + command = "bash ${path.module}/scripts/create.sh" + + environment = { + CONTAINER_APP_NAME = self.triggers.ca_name + CONTAINER_APP_RESOURCE_GROUP = self.triggers.ca_rg_name + CONTAINER_APP_ENV_NAME = self.triggers.ca_env_name + CONTAINER_APP_ENV_RESOURCE_GROUP = self.triggers.ca_env_rg_name + CUSTOM_DOMAIN = self.triggers.custom_domain + } + } + + provisioner "local-exec" { + when = destroy + command = "bash ${path.module}/scripts/destroy.sh" + + environment = { + CONTAINER_APP_NAME = self.triggers.ca_name + CONTAINER_APP_RESOURCE_GROUP = self.triggers.ca_rg_name + CONTAINER_APP_ENV_NAME = self.triggers.ca_env_name + CONTAINER_APP_ENV_RESOURCE_GROUP = self.triggers.ca_env_rg_name + CUSTOM_DOMAIN = self.triggers.custom_domain + } + } +} \ No newline at end of file diff --git a/terraform/instance/container_apps_bind_dns/scripts/create.sh b/terraform/instance/container_apps_bind_dns/scripts/create.sh new file mode 100644 index 0000000..222279c --- /dev/null +++ b/terraform/instance/container_apps_bind_dns/scripts/create.sh @@ -0,0 +1,124 @@ +#!/bin/bash + +# env variables used throughout this script: +# CONTAINER_APP_NAME +# CONTAINER_APP_RESOURCE_GROUP +# CONTAINER_APP_ENV_NAME +# CONTAINER_APP_ENV_RESOURCE_GROUP +# CUSTOM_DOMAIN + + +# functions below taken from: https://stackoverflow.com/a/25515370 +yell() { echo "$0: $*" >&2; } +die() { + yell "$*" + exit 111 +} + +# use dig to verify the asuid txt record exists on the DNS host +# azure requires this to exist prior to adding the domain +# azure's dns can also be slow, so best to check propagation +tries=0 +until [ "$tries" -ge 12 ]; do + [[ ! -z $(dig @8.8.8.8 txt asuid.$CUSTOM_DOMAIN +short) ]] && break + tries=$((tries + 1)) + sleep 10 +done +if [ "$tries" -ge 12 ]; then + die "'asuid.${CUSTOM_DOMAIN}' txt record does not exist" +fi + +echo "took $tries trie(s) for the dns record to exist publically" + +# check if the hostname already exists on the container app +# if not, add it since it's required to provision a managed cert +DOES_CUSTOM_DOMAIN_EXIST=$( + az containerapp hostname list \ + -n $CONTAINER_APP_NAME \ + -g $CONTAINER_APP_RESOURCE_GROUP \ + --query "[?name=='$CUSTOM_DOMAIN'].name" \ + --output tsv +) +if [ -z "${DOES_CUSTOM_DOMAIN_EXIST}" ]; then + echo "adding custom hostname to container app first since it does not exist yet" + az containerapp hostname add \ + -n $CONTAINER_APP_NAME \ + -g $CONTAINER_APP_RESOURCE_GROUP \ + --hostname $CUSTOM_DOMAIN \ + --output none +fi + +# check if a managed cert for the domain already exists +# if it does not exist, provision one +# if it does, save its name to use for binding it later +MANAGED_CERTIFICATE_ID=$( + az containerapp env certificate list \ + -g $CONTAINER_APP_ENV_RESOURCE_GROUP \ + -n $CONTAINER_APP_ENV_NAME \ + --managed-certificates-only \ + --query "[?properties.subjectName=='$CUSTOM_DOMAIN'].id" \ + --output tsv +) +if [ -z "${MANAGED_CERTIFICATE_ID}" ]; then + MANAGED_CERTIFICATE_ID=$( + az containerapp env certificate create \ + -g $CONTAINER_APP_ENV_RESOURCE_GROUP \ + -n $CONTAINER_APP_ENV_NAME \ + --hostname $CUSTOM_DOMAIN \ + --validation-method CNAME \ + --query "id" \ + --output tsv + ) + echo "created cert for '$CUSTOM_DOMAIN'. waiting for it to provision now..." + + # poll azcli to check for the certificate status + # this is better than waiting 5 minutes, because it could be + # faster and we get to exit the script faster + # --- + # the default 20 tries means it'll check for 5 mins + # at 15 second intervals + tries=0 + until [ "$tries" -ge 20 ]; do + STATE=$( + az containerapp env certificate list \ + -g $CONTAINER_APP_ENV_RESOURCE_GROUP \ + -n $CONTAINER_APP_ENV_NAME \ + --managed-certificates-only \ + --query "[?properties.subjectName=='$CUSTOM_DOMAIN'].properties.provisioningState" \ + --output tsv + ) + [[ $STATE == "Succeeded" ]] && break + tries=$((tries + 1)) + + sleep 15 + done + if [ "$tries" -ge 20 ]; then + die "waited for 5 minutes, checked the certificate status 20 times and its not done. check azure portal..." + fi +else + echo "found existing cert in the env. proceeding to use that" +fi + +# check if the cert has already been bound +# if not, bind it then +IS_CERT_ALREADY_BOUND=$( + az containerapp hostname list \ + -n $CONTAINER_APP_NAME \ + -g $CONTAINER_APP_RESOURCE_GROUP \ + --query "[?name=='$CUSTOM_DOMAIN'].bindingType" \ + --output tsv +) +if [ $IS_CERT_ALREADY_BOUND = "SniEnabled" ]; then + echo "cert is already bound, exiting..." +else + # try bind the cert to the container app + echo "cert successfully provisioned. binding the cert id to the hostname" + az containerapp hostname bind \ + -g $CONTAINER_APP_RESOURCE_GROUP \ + -n $CONTAINER_APP_NAME \ + --hostname $CUSTOM_DOMAIN \ + --environment $CONTAINER_APP_ENV_NAME \ + --certificate $MANAGED_CERTIFICATE_ID \ + --output none + echo "finished binding. the domain is now secured and ready to use" +fi diff --git a/terraform/instance/container_apps_bind_dns/scripts/destroy.sh b/terraform/instance/container_apps_bind_dns/scripts/destroy.sh new file mode 100644 index 0000000..df31b37 --- /dev/null +++ b/terraform/instance/container_apps_bind_dns/scripts/destroy.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# functions below taken from: https://stackoverflow.com/a/25515370 +yell() { echo "$0: $*" >&2; } +die() { + yell "$*" + exit 111 +} + +# get the managed cert using the custom domain +CERTIFICATE_ID=$( + az containerapp env certificate list \ + -g $CONTAINER_APP_ENV_RESOURCE_GROUP \ + -n $CONTAINER_APP_ENV_NAME \ + --managed-certificates-only \ + --query "[?properties.subjectName=='$CUSTOM_DOMAIN'].id" \ + --output tsv +) + +# destroy the cert +az containerapp env certificate delete \ + -g $CONTAINER_APP_ENV_RESOURCE_GROUP \ + -n $CONTAINER_APP_ENV_NAME \ + --certificate $CERTIFICATE_ID --yes +echo "destroyed the managed certificate" + +# remove the custom domain from the container app +az containerapp hostname delete --hostname $CUSTOM_DOMAIN \ + -g $CONTAINER_APP_RESOURCE_GROUP \ + -n $CONTAINER_APP_NAME +echo "removed the custom domain from the container app" diff --git a/terraform/instance/container_apps_bind_dns/variables.tf b/terraform/instance/container_apps_bind_dns/variables.tf new file mode 100644 index 0000000..182175f --- /dev/null +++ b/terraform/instance/container_apps_bind_dns/variables.tf @@ -0,0 +1,22 @@ +variable "container_app_resource_group_name" { + description = "name of the resource group where the container apps are deployed" + type = string +} + +variable "container_app_env_name" { + description = "name of the container app environment name" + type = string +} + +variable "container_app_env_resource_group_name" { + description = "name of the resource group where the container app environment is deployed" + type = string +} + +variable "services" { + type = list(object({ + key = string + custom_domain = string + container_app_name = string + })) +} \ No newline at end of file diff --git a/terraform/instance/dns.tf b/terraform/instance/dns.tf index d67b0a3..f85b817 100644 --- a/terraform/instance/dns.tf +++ b/terraform/instance/dns.tf @@ -1,6 +1,6 @@ -data "azurerm_dns_zone" "rockpal-co-uk"{ - resource_group_name = "onlinestore-shared-rg" - name = "rockpal.co.uk" +data "azurerm_dns_zone" "rockpal-co-uk" { + resource_group_name = "onlinestore-shared-rg" + name = "rockpal.co.uk" } resource "azurerm_dns_cname_record" "api" { @@ -21,21 +21,6 @@ resource "azurerm_dns_txt_record" "api" { } } -resource "azurerm_container_app_custom_domain" "api" { - name = trimsuffix(trimprefix(azurerm_dns_txt_record.api.fqdn, "asuid."), ".") - container_app_id = azurerm_container_app.api.id - certificate_binding_type = "SniEnabled" - - depends_on = [ - azurerm_dns_txt_record.api, - ] - - lifecycle { - // When using an Azure created Managed Certificate these values must be added to ignore_changes to prevent resource recreation. - ignore_changes = [certificate_binding_type, container_app_environment_certificate_id] - } -} - resource "azurerm_dns_cname_record" "website" { name = var.website_dns_subdomain zone_name = data.azurerm_dns_zone.rockpal-co-uk.name @@ -54,21 +39,6 @@ resource "azurerm_dns_txt_record" "website" { } } -resource "azurerm_container_app_custom_domain" "website" { - name = trimsuffix(trimprefix(azurerm_dns_txt_record.website.fqdn, "asuid."), ".") - container_app_id = azurerm_container_app.website.id - certificate_binding_type = "SniEnabled" - - depends_on = [ - azurerm_dns_txt_record.website, - ] - - lifecycle { - // When using an Azure created Managed Certificate these values must be added to ignore_changes to prevent resource recreation. - ignore_changes = [certificate_binding_type, container_app_environment_certificate_id] - } -} - resource "azurerm_dns_cname_record" "monitoring" { name = var.monitoring_dns_subdomain zone_name = data.azurerm_dns_zone.rockpal-co-uk.name @@ -87,17 +57,26 @@ resource "azurerm_dns_txt_record" "monitoring" { } } -resource "azurerm_container_app_custom_domain" "monitoring" { - name = trimsuffix(trimprefix(azurerm_dns_txt_record.monitoring.fqdn, "asuid."), ".") - container_app_id = azurerm_container_app.monitoring.id - certificate_binding_type = "SniEnabled" - - depends_on = [ - azurerm_dns_txt_record.monitoring, +module "container_apps_bind_dns" { + source = "./container_apps_bind_dns" + container_app_resource_group_name = data.azurerm_container_app_environment.apps.resource_group_name + container_app_env_name = data.azurerm_container_app_environment.apps.name + container_app_env_resource_group_name = data.azurerm_container_app_environment.apps.resource_group_name + services = [ + { + key = "api", + custom_domain = trimsuffix(azurerm_dns_cname_record.api.fqdn, "."), + container_app_name = azurerm_container_app.api.name + }, + { + key = "website", + custom_domain = trimsuffix(azurerm_dns_cname_record.website.fqdn, "."), + container_app_name = azurerm_container_app.website.name + }, + { + key = "monitoring", + custom_domain = trimsuffix(azurerm_dns_cname_record.monitoring.fqdn, "."), + container_app_name = azurerm_container_app.monitoring.name + } ] - - lifecycle { - // When using an Azure created Managed Certificate these values must be added to ignore_changes to prevent resource recreation. - ignore_changes = [certificate_binding_type, container_app_environment_certificate_id] - } } diff --git a/terraform/instance/main.tf b/terraform/instance/main.tf index f792457..0097f17 100644 --- a/terraform/instance/main.tf +++ b/terraform/instance/main.tf @@ -32,18 +32,3 @@ output "resource_group_name" { value = azurerm_resource_group.instance.name sensitive = false } - -output "container_app_api_fqdn" { - value = azurerm_container_app.api.ingress[0].fqdn - sensitive = false -} - -output "container_app_website_fqdn" { - value = azurerm_container_app.website.ingress[0].fqdn - sensitive = false -} - -output "container_app_monitoring_fqdn" { - value = azurerm_container_app.monitoring.ingress[0].fqdn - sensitive = false -}