Resolve DNS issues on container apps

.github/workflows/instance-deploy-prod.yml

@@ -80,9 +80,6 @@ jobs:
         run: |
           mkdir terraform-outputs
           terraform -chdir=instance output -raw resource_group_name > terraform-outputs/resource_group_name.txt
-          terraform -chdir=instance output -raw container_app_api_fqdn > terraform-outputs/container_app_api_fqdn.txt
-          terraform -chdir=instance output -raw container_app_website_fqdn > terraform-outputs/container_app_website_fqdn.txt
-          terraform -chdir=instance show -json | jq -r '.values.outputs.container_app_monitoring_fqdn.value // ""' > container_app_monitoring_fqdn.txt
       - name: Upload terraform outputs for deploy job
         uses: actions/upload-artifact@v3
         with:
@@ -105,34 +102,29 @@ jobs:
         shell: bash
         run: |
           echo "resource_group_name=$(cat resource_group_name.txt)" >> $GITHUB_ENV
-          echo "container_app_api_fqdn=$(cat container_app_api_fqdn.txt)" >> $GITHUB_ENV
-          echo "container_app_website_fqdn=$(cat container_app_website_fqdn.txt)" >> $GITHUB_ENV
-          echo "container_app_monitoring_fqdn=$(cat container_app_monitoring_fqdn.txt)" >> $GITHUB_ENV
       - name: Login via Azure CLI
         uses: azure/login@v1
         with:
           creds: ${{ secrets.AZURE_CREDENTIALS }}
       - name: Deploy api
         uses: azure/container-apps-deploy-action@v1
         with:
-          yamlConfigPath: .github/workflows/config/container-environment-config.yaml
           acrName: onlinestorecontainerregistry
           acrUsername: ${{ secrets.ACR_USERNAME }}
           acrPassword: ${{ secrets.ACR_TOKEN }}
-          containerAppName: onlinestore-api
+          containerAppName: prod-onlinestore-api
           imageToDeploy: onlinestorecontainerregistry.azurecr.io/onlinestore-api:${{ github.sha }}
           location: 'East US'
           resourceGroup: ${{ env.resource_group_name }}
           targetPort: 8080
-          environmentVariables: OTEL_EXPORTER_OTLP_ENDPOINT=http://onlinestore-monitoring:18889
+          environmentVariables: OTEL_EXPORTER_OTLP_ENDPOINT=http://prod-onlinestore-monitoring:18889
       - name: Deploy website
         uses: azure/container-apps-deploy-action@v1
         with:
-          yamlConfigPath: .github/workflows/config/container-environment-config.yaml
           acrName: onlinestorecontainerregistry
           acrUsername: ${{ secrets.ACR_USERNAME }}
           acrPassword: ${{ secrets.ACR_TOKEN }}
-          containerAppName: onlinestore-website
+          containerAppName: prod-onlinestore-website
           imageToDeploy: onlinestorecontainerregistry.azurecr.io/onlinestore-website:${{ github.sha }}
           location: 'East US'
           resourceGroup: ${{ env.resource_group_name }}

.github/workflows/instance-deploy-test.yml

@@ -43,6 +43,10 @@ jobs:
         uses: hashicorp/setup-terraform@v2
         with:
           terraform_wrapper: false
+      - name: Login via Azure CLI
+        uses: azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
       - name: Terraform Format
         id: fmt
         run: terraform -chdir=instance fmt
@@ -60,7 +64,7 @@ jobs:
           TF_VAR_environment: ${{ github.head_ref }}
           TF_VAR_acr_username: ${{ secrets.ACR_USERNAME }}
           TF_VAR_acr_password: ${{ secrets.ACR_TOKEN }}
-          TF_VAR_website_dns_subdomain: ${{ github.head_ref }}-site
+          TF_VAR_website_dns_subdomain: ${{ github.head_ref }}-website
           TF_VAR_api_dns_subdomain: ${{ github.head_ref }}-api
           TF_VAR_monitoring_dns_subdomain: ${{ github.head_ref }}-monitoring
       - name: Terraform Apply
@@ -71,17 +75,14 @@ jobs:
           TF_VAR_environment: ${{ github.head_ref }}
           TF_VAR_acr_username: ${{ secrets.ACR_USERNAME }}
           TF_VAR_acr_password: ${{ secrets.ACR_TOKEN }}
-          TF_VAR_website_dns_subdomain: ${{ github.head_ref }}
-          TF_VAR_api_dns_subdomain: api
-          TF_VAR_monitoring_dns_subdomain: monitoring
+          TF_VAR_website_dns_subdomain: ${{ github.head_ref }}-website
+          TF_VAR_api_dns_subdomain: ${{ github.head_ref }}-api
+          TF_VAR_monitoring_dns_subdomain: ${{ github.head_ref }}-monitoring
       - name: Save terraform outputs
         shell: bash
         run: |
           mkdir terraform-outputs
           terraform -chdir=instance output -raw resource_group_name > terraform-outputs/resource_group_name.txt
-          terraform -chdir=instance output -raw container_app_api_fqdn > terraform-outputs/container_app_api_fqdn.txt
-          terraform -chdir=instance output -raw container_app_website_fqdn > terraform-outputs/container_app_website_fqdn.txt
-          terraform -chdir=instance output -raw container_app_monitoring_fqdn > terraform-outputs/container_app_monitoring_fqdn.txt
       - name: Upload terraform outputs for deploy job
         uses: actions/upload-artifact@v3
         with:
@@ -104,9 +105,6 @@ jobs:
         shell: bash
         run: |
           echo "resource_group_name=$(cat resource_group_name.txt)" >> $GITHUB_ENV
-          echo "container_app_api_fqdn=$(cat container_app_api_fqdn.txt)" >> $GITHUB_ENV
-          echo "container_app_website_fqdn=$(cat container_app_website_fqdn.txt)" >> $GITHUB_ENV
-          echo "container_app_monitoring_fqdn=$(cat container_app_monitoring_fqdn.txt)" >> $GITHUB_ENV
       - name: Login via Azure CLI
         uses: azure/login@v1
         with:
@@ -117,22 +115,22 @@ jobs:
           acrName: onlinestorecontainerregistry
           acrUsername: ${{ secrets.ACR_USERNAME }}
           acrPassword: ${{ secrets.ACR_TOKEN }}
-          containerAppName: onlinestore-api
+          containerAppName: ${{ github.head_ref }}-onlinestore-api
           imageToDeploy: onlinestorecontainerregistry.azurecr.io/onlinestore-api:${{ github.sha }}
           location: 'East US'
-          resourceGroup: ${{ env.resource_group_name }}
+          resourceGroup: onlinestore-shared-rg
           targetPort: 8080
-          environmentVariables: OTEL_EXPORTER_OTLP_ENDPOINT=http://onlinestore-monitoring:18889
+          environmentVariables: OTEL_EXPORTER_OTLP_ENDPOINT=http://${{ github.head_ref }}-onlinestore-monitoring:18889
       - name: Deploy website
         uses: azure/container-apps-deploy-action@v1
         with:
           acrName: onlinestorecontainerregistry
           acrUsername: ${{ secrets.ACR_USERNAME }}
           acrPassword: ${{ secrets.ACR_TOKEN }}
-          containerAppName: onlinestore-website
+          containerAppName: ${{ github.head_ref }}-onlinestore-website
           imageToDeploy: onlinestorecontainerregistry.azurecr.io/onlinestore-website:${{ github.sha }}
           location: 'East US'
-          resourceGroup: ${{ env.resource_group_name }}
+          resourceGroup: onlinestore-shared-rg
           targetPort: 80
           environmentVariables: "API__BASEPATH=https://${{ github.head_ref }}-api.rockpal.co.uk"
       - name: Find Comment

.github/workflows/instance-destroy-test.yaml → .github/workflows/instance-destroy-test.yml

@@ -31,3 +31,5 @@ jobs:
           TF_VAR_acr_username: "not-used"
           TF_VAR_acr_password: "not-used"
           TF_VAR_website_dns_subdomain: "not-used"
+          TF_VAR_api_dns_subdomain: "not-used"
+          TF_VAR_monitoring_dns_subdomain: "not-used"

terraform/instance/container_apps.tf

@@ -4,9 +4,9 @@ data  "azurerm_container_app_environment" "apps" {
 }
 
 resource "azurerm_container_app" "api" {
-  name                         = "${var.name}-api"
+  name                         = "${lower(var.environment)}-${var.name}-api"
   container_app_environment_id = data.azurerm_container_app_environment.apps.id
-  resource_group_name          = azurerm_resource_group.instance.name
+  resource_group_name          = data.azurerm_container_app_environment.apps.resource_group_name
   revision_mode                = "Single"
 
   template {
@@ -39,9 +39,9 @@ resource "azurerm_container_app" "api" {
 }
 
 resource "azurerm_container_app" "website" {
-  name                         = "${var.name}-website"
+  name                         = "${lower(var.environment)}-${var.name}-website"
   container_app_environment_id = data.azurerm_container_app_environment.apps.id
-  resource_group_name          = azurerm_resource_group.instance.name
+  resource_group_name          = data.azurerm_container_app_environment.apps.resource_group_name
   revision_mode                = "Single"
 
   template {
@@ -74,9 +74,9 @@ resource "azurerm_container_app" "website" {
 }
 
 resource "azurerm_container_app" "monitoring" {
-  name                         = "${var.name}-monitoring"
+  name                         = "${lower(var.environment)}-${var.name}-monitoring"
   container_app_environment_id = data.azurerm_container_app_environment.apps.id
-  resource_group_name          = azurerm_resource_group.instance.name
+  resource_group_name          = data.azurerm_container_app_environment.apps.resource_group_name
   revision_mode                = "Single"
 
   template {

terraform/instance/container_apps_bind_dns/main.tf

@@ -0,0 +1,44 @@
+terraform {}
+
+resource "null_resource" "null" {
+  for_each = { for svc in var.services : svc.key => svc }
+
+  lifecycle {
+    create_before_destroy = false
+  }
+
+  triggers = {
+    ca_name        = each.value.container_app_name
+    ca_rg_name     = var.container_app_resource_group_name
+    ca_env_name    = var.container_app_env_name
+    ca_env_rg_name = var.container_app_env_resource_group_name
+    custom_domain  = each.value.custom_domain
+  }
+
+  # provision a managed cert and apply it to the container app
+  provisioner "local-exec" {
+    when    = create
+    command = "bash ${path.module}/scripts/create.sh"
+
+    environment = {
+      CONTAINER_APP_NAME               = self.triggers.ca_name
+      CONTAINER_APP_RESOURCE_GROUP     = self.triggers.ca_rg_name
+      CONTAINER_APP_ENV_NAME           = self.triggers.ca_env_name
+      CONTAINER_APP_ENV_RESOURCE_GROUP = self.triggers.ca_env_rg_name
+      CUSTOM_DOMAIN                    = self.triggers.custom_domain
+    }
+  }
+
+  provisioner "local-exec" {
+    when    = destroy
+    command = "bash ${path.module}/scripts/destroy.sh"
+
+    environment = {
+      CONTAINER_APP_NAME               = self.triggers.ca_name
+      CONTAINER_APP_RESOURCE_GROUP     = self.triggers.ca_rg_name
+      CONTAINER_APP_ENV_NAME           = self.triggers.ca_env_name
+      CONTAINER_APP_ENV_RESOURCE_GROUP = self.triggers.ca_env_rg_name
+      CUSTOM_DOMAIN                    = self.triggers.custom_domain
+    }
+  }
+}

terraform/instance/container_apps_bind_dns/scripts/create.sh

@@ -0,0 +1,124 @@
+#!/bin/bash
+
+# env variables used throughout this script:
+# CONTAINER_APP_NAME
+# CONTAINER_APP_RESOURCE_GROUP
+# CONTAINER_APP_ENV_NAME
+# CONTAINER_APP_ENV_RESOURCE_GROUP
+# CUSTOM_DOMAIN
+
+
+# functions below taken from: https://stackoverflow.com/a/25515370
+yell() { echo "$0: $*" >&2; }
+die() {
+  yell "$*"
+  exit 111
+}
+
+# use dig to verify the asuid txt record exists on the DNS host
+# azure requires this to exist prior to adding the domain
+# azure's dns can also be slow, so best to check propagation
+tries=0
+until [ "$tries" -ge 12 ]; do
+  [[ ! -z $(dig @8.8.8.8 txt asuid.$CUSTOM_DOMAIN +short) ]] && break
+  tries=$((tries + 1))
+  sleep 10
+done
+if [ "$tries" -ge 12 ]; then
+  die "'asuid.${CUSTOM_DOMAIN}' txt record does not exist"
+fi
+
+echo "took $tries trie(s) for the dns record to exist publically"
+
+# check if the hostname already exists on the container app
+# if not, add it since it's required to provision a managed cert
+DOES_CUSTOM_DOMAIN_EXIST=$(
+  az containerapp hostname list \
+    -n $CONTAINER_APP_NAME \
+    -g $CONTAINER_APP_RESOURCE_GROUP \
+    --query "[?name=='$CUSTOM_DOMAIN'].name" \
+    --output tsv
+)
+if [ -z "${DOES_CUSTOM_DOMAIN_EXIST}" ]; then
+  echo "adding custom hostname to container app first since it does not exist yet"
+  az containerapp hostname add \
+    -n $CONTAINER_APP_NAME \
+    -g $CONTAINER_APP_RESOURCE_GROUP \
+    --hostname $CUSTOM_DOMAIN \
+    --output none
+fi
+
+# check if a managed cert for the domain already exists
+# if it does not exist, provision one
+# if it does, save its name to use for binding it later
+MANAGED_CERTIFICATE_ID=$(
+  az containerapp env certificate list \
+    -g $CONTAINER_APP_ENV_RESOURCE_GROUP \
+    -n $CONTAINER_APP_ENV_NAME \
+    --managed-certificates-only \
+    --query "[?properties.subjectName=='$CUSTOM_DOMAIN'].id" \
+    --output tsv
+)
+if [ -z "${MANAGED_CERTIFICATE_ID}" ]; then
+  MANAGED_CERTIFICATE_ID=$(
+    az containerapp env certificate create \
+      -g $CONTAINER_APP_ENV_RESOURCE_GROUP \
+      -n $CONTAINER_APP_ENV_NAME \
+      --hostname $CUSTOM_DOMAIN \
+      --validation-method CNAME \
+      --query "id" \
+      --output tsv
+  )
+  echo "created cert for '$CUSTOM_DOMAIN'. waiting for it to provision now..."
+
+  # poll azcli to check for the certificate status
+  # this is better than waiting 5 minutes, because it could be
+  # faster and we get to exit the script faster
+  # ---
+  # the default 20 tries means it'll check for 5 mins
+  # at 15 second intervals
+  tries=0
+  until [ "$tries" -ge 20 ]; do
+    STATE=$(
+      az containerapp env certificate list \
+        -g $CONTAINER_APP_ENV_RESOURCE_GROUP \
+        -n $CONTAINER_APP_ENV_NAME \
+        --managed-certificates-only \
+        --query "[?properties.subjectName=='$CUSTOM_DOMAIN'].properties.provisioningState" \
+        --output tsv
+    )
+    [[ $STATE == "Succeeded" ]] && break
+    tries=$((tries + 1))
+
+    sleep 15
+  done
+  if [ "$tries" -ge 20 ]; then
+    die "waited for 5 minutes, checked the certificate status 20 times and its not done. check azure portal..."
+  fi
+else
+  echo "found existing cert in the env. proceeding to use that"
+fi
+
+# check if the cert has already been bound
+# if not, bind it then
+IS_CERT_ALREADY_BOUND=$(
+  az containerapp hostname list \
+    -n $CONTAINER_APP_NAME \
+    -g $CONTAINER_APP_RESOURCE_GROUP \
+    --query "[?name=='$CUSTOM_DOMAIN'].bindingType" \
+    --output tsv
+)
+if [ $IS_CERT_ALREADY_BOUND = "SniEnabled" ]; then
+  echo "cert is already bound, exiting..."
+else
+  # try bind the cert to the container app
+  echo "cert successfully provisioned. binding the cert id to the hostname"
+  az containerapp hostname bind \
+    -g $CONTAINER_APP_RESOURCE_GROUP \
+    -n $CONTAINER_APP_NAME \
+    --hostname $CUSTOM_DOMAIN \
+    --environment $CONTAINER_APP_ENV_NAME \
+    --certificate $MANAGED_CERTIFICATE_ID \
+    --output none
+  echo "finished binding. the domain is now secured and ready to use"
+fi

terraform/instance/container_apps_bind_dns/scripts/destroy.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# functions below taken from: https://stackoverflow.com/a/25515370
+yell() { echo "$0: $*" >&2; }
+die() {
+  yell "$*"
+  exit 111
+}
+
+# get the managed cert using the custom domain
+CERTIFICATE_ID=$(
+  az containerapp env certificate list \
+    -g $CONTAINER_APP_ENV_RESOURCE_GROUP \
+    -n $CONTAINER_APP_ENV_NAME \
+    --managed-certificates-only \
+    --query "[?properties.subjectName=='$CUSTOM_DOMAIN'].id" \
+    --output tsv
+)
+
+# destroy the cert
+az containerapp env certificate delete \
+  -g $CONTAINER_APP_ENV_RESOURCE_GROUP \
+  -n $CONTAINER_APP_ENV_NAME \
+  --certificate $CERTIFICATE_ID --yes
+echo "destroyed the managed certificate"
+
+# remove the custom domain from the container app
+az containerapp hostname delete --hostname $CUSTOM_DOMAIN \
+  -g $CONTAINER_APP_RESOURCE_GROUP \
+  -n $CONTAINER_APP_NAME
+echo "removed the custom domain from the container app"

terraform/instance/container_apps_bind_dns/variables.tf

@@ -0,0 +1,22 @@
+variable "container_app_resource_group_name" {
+  description = "name of the resource group where the container apps are deployed"
+  type = string
+}
+
+variable "container_app_env_name" {
+  description = "name of the container app environment name"
+  type = string
+}
+
+variable "container_app_env_resource_group_name" {
+  description = "name of the resource group where the container app environment is deployed"
+  type = string
+}
+
+variable "services" {
+  type = list(object({
+    key = string
+    custom_domain = string
+    container_app_name = string
+  }))
+}