Dynamically assign Aurora CIDR range based upon existing Aurora DB in…

…stances Fixes #571 Signed-off-by: Ryan Emerson <[email protected]> Co-authored-by: Alexander Schwartz <[email protected]>
keycloak · Dec 15, 2023 · c59a5c2 · c59a5c2
1 parent 7dc1a7b
commit c59a5c2
Show file tree

Hide file tree

Showing 12 changed files with 96 additions and 74 deletions.
diff --git a/doc/kubernetes/modules/ROOT/pages/storage/aurora-regional-postgres.adoc b/doc/kubernetes/modules/ROOT/pages/storage/aurora-regional-postgres.adoc
@@ -45,7 +45,7 @@ CLUSTER_NAME= # The name of the ROSA cluster to establish the peering connectin
 AWS_REGION= # The AWS region hosting the ROSA cluster
 ----
 
-NOTE: If connecting many ROSA clusters from different AWS regions to the Aurora DB, it is necessary for each cluster to have non-overlapping machine-CIDR configured.
+The scripts in this project ensure that all Aurora and ROSA clusters have non-overlapping machine-CIDR configured.
 
 == Enabling Aurora PostgreSQL storage
 

diff --git a/provision/aws/rds/aurora_cluster_reaper.sh b/provision/aws/rds/aurora_cluster_reaper.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+set -e
 
 if [[ "$RUNNER_DEBUG" == "1" ]]; then
   set -x
@@ -81,7 +82,7 @@ for REGION in ${REGIONS}; do
 
             KEEP_ALIVE=$(keepAlive ${REGION} ${AURORA_CLUSTER})
             if [ ${KEEP_ALIVE} == "0" ]; then
-                export AWS_REGION=${REGION}
+                export AURORA_REGION=${REGION}
                 export RUNNER_DEBUG=1
                 unset AURORA_SECURITY_GROUP_NAME
                 unset AURORA_SUBNET_GROUP_NAME

diff --git a/provision/aws/rds/aurora_common.sh b/provision/aws/rds/aurora_common.sh
@@ -9,10 +9,7 @@ export AURORA_INSTANCE_CLASS=${AURORA_INSTANCE_CLASS:-"db.t4g.large"}
 export AURORA_PASSWORD=${AURORA_PASSWORD:-"secret99"}
 export AURORA_REGION=${AURORA_REGION}
 export AURORA_SECURITY_GROUP_NAME=${AURORA_SECURITY_GROUP_NAME:-"${AURORA_CLUSTER}-security-group"}
-export AURORA_SUBNET_A_CIDR=${AURORA_SUBNET_A_CIDR:-"192.168.0.0/19"}
-export AURORA_SUBNET_B_CIDR=${AURORA_SUBNET_B_CIDR:-"192.168.32.0/19"}
 export AURORA_SUBNET_GROUP_NAME=${AURORA_SUBNET_GROUP_NAME:-"${AURORA_CLUSTER}-subnet-group"}
 export AURORA_USERNAME=${AURORA_USERNAME:-"keycloak"}
-export AURORA_VPC_CIDR=${AURORA_VPC_CIDR:-"192.168.0.0/16"}
 export AWS_REGION=${AWS_REGION:-${AURORA_REGION}}
 export AWS_PAGER=""
diff --git a/provision/aws/rds/aurora_create.sh b/provision/aws/rds/aurora_create.sh
@@ -18,6 +18,40 @@ if [ -n "${EXISTING_INSTANCES}" ]; then
   exit 0
 fi
 
+REGIONS=$(aws ec2 describe-regions \
+    --query "Regions[*].RegionName" \
+    --output text
+)
+
+EXISTING_AURORA_CIDRS=""
+for REGION in ${REGIONS}; do
+  CIDRS=$(aws ec2 describe-vpcs \
+    --filters Name=tag-key,Values=AuroraCluster \
+    --query "Vpcs[*].CidrBlock" \
+    --region ${REGION} \
+    --output text
+  )
+  if [[ "${EXISTING_AURORA_CIDRS}" != *"${CIDRS}"* ]]; then
+    EXISTING_AURORA_CIDRS+=" ${CIDRS}"
+  fi
+done
+
+if (( $(echo ${EXISTING_AURORA_CIDRS} | wc -l) > 63 )); then
+  echo "Maximum number of unique Aurora CIDRS reached"
+  echo ${EXISTING_AURORA_CIDRS}
+  exit 1
+fi
+
+while true; do
+  AURORA_VPC_RANGE="192.168.$(shuf -i 0-255 -n 1)"
+  AURORA_VPC_CIDR="${AURORA_VPC_RANGE}.0/24"
+  if [[ "${EXISTING_MACHINE_CIDRS}" != *"${AURORA_VPC_CIDR}"* ]]; then
+    break
+  fi
+done
+AURORA_SUBNET_A_CIDR=${AURORA_VPC_RANGE}.0/25 # 0-127
+AURORA_SUBNET_B_CIDR=${AURORA_VPC_RANGE}.128/25 # 128-255
+
 # Create the Aurora VPC
 AURORA_VPC=$(aws ec2 create-vpc \
   --cidr-block ${AURORA_VPC_CIDR} \

diff --git a/provision/aws/rds/aurora_create_global_db.sh b/provision/aws/rds/aurora_create_global_db.sh
@@ -44,9 +44,6 @@ for (( i = 0 ; i < ${#GLOBAL_REGIONS[@]} ; i++ )) ; do
   # Aurora Global DBs must use one of the memory optimized classes
   export AURORA_INSTANCE_CLASS="db.r5.large"
   export AURORA_REGION=${REGION}
-  export AURORA_VPC_CIDR=$(globalAuroraVpcCidr $i)
-  export AURORA_SUBNET_A_CIDR=$(globalAuroraSubnetA $i)
-  export AURORA_SUBNET_B_CIDR=$(globalAuroraSubnetB $i)
 
   if [ "${REGION}" != "${PRIMARY_REGION}" ]; then
     export AURORA_GLOBAL_CLUSTER_BACKUP=true

diff --git a/provision/aws/rds/aurora_create_global_peering_connections.sh b/provision/aws/rds/aurora_create_global_peering_connections.sh
@@ -19,6 +19,5 @@ for (( i = 0 ; i < ${#GLOBAL_REGIONS[@]} ; i++ )) ; do
   REGION=${GLOBAL_REGIONS[i]}
   export AURORA_CLUSTER=${AURORA_GLOBAL_CLUSTER}-${REGION}
   export AURORA_REGION=${REGION}
-  export AURORA_VPC_CIDR=$(globalAuroraVpcCidr $i)
   ${SCRIPT_DIR}/aurora_create_peering_connection.sh || true
 done
diff --git a/provision/aws/rds/aurora_create_peering_connection.sh b/provision/aws/rds/aurora_create_peering_connection.sh
@@ -17,11 +17,13 @@ ROSA_MACHINE_CIDR=$(echo ${ROSA_CLUSTER} | jq -r .network.machine_cidr)
 export AWS_REGION=$(echo ${ROSA_CLUSTER} | jq -r .region.id)
 
 AURORA_VPC=$(aws ec2 describe-vpcs \
-  --filters "Name=cidr,Values=${AURORA_VPC_CIDR}" "Name=tag:AuroraCluster,Values=${AURORA_CLUSTER}" \
-  --query 'Vpcs[*].VpcId' \
+  --filters "Name=tag:AuroraCluster,Values=${AURORA_CLUSTER}" \
+  --query 'Vpcs[0]' \
+  --output json \
   --region ${AURORA_REGION} \
-  --output text
 )
+AURORA_VPC_ID=$(echo ${AURORA_VPC} | jq -r .VpcId)
+AURORA_VPC_CIDR=$(echo ${AURORA_VPC} | jq -r .CidrBlock)
 
 NODE=$(oc get nodes --selector=node-role.kubernetes.io/worker \
   -o jsonpath='{.items[0].metadata.name}'
@@ -36,15 +38,15 @@ ROSA_VPC=$(aws ec2 describe-instances \
 
 # Create and Accept a VPC Peering Connection between Aurora and a ROSA cluster's VPC if it doesn't already exist
 PEERING_CONNECTION_ID=$(aws ec2 describe-vpc-peering-connections \
-  --filter "Name=requester-vpc-info.vpc-id,Values=${ROSA_VPC}" "Name=requester-vpc-info.vpc-id,Values=${AURORA_VPC}" "Name=status-code,Values=active"\
+  --filter "Name=requester-vpc-info.vpc-id,Values=${ROSA_VPC}" "Name=requester-vpc-info.vpc-id,Values=${AURORA_VPC_ID}" "Name=status-code,Values=active"\
   --query "VpcPeeringConnections[*].VpcPeeringConnectionId" \
   --output text
 )
 
 if [ -z "${PEERING_CONNECTION_ID}" ]; then
   PEERING_CONNECTION_ID=$(aws ec2 create-vpc-peering-connection \
     --vpc-id ${ROSA_VPC} \
-    --peer-vpc-id ${AURORA_VPC} \
+    --peer-vpc-id ${AURORA_VPC_ID} \
     --peer-region ${AURORA_REGION} \
     --query VpcPeeringConnection.VpcPeeringConnectionId \
     --output text
@@ -85,7 +87,7 @@ fi
 
 # Update the Aurora Cluster VPC's Route Table
 AURORA_PUBLIC_ROUTE_TABLE_ID=$(aws ec2 describe-route-tables \
-  --filters "Name=vpc-id,Values=${AURORA_VPC}" "Name=association.main,Values=true" \
+  --filters "Name=vpc-id,Values=${AURORA_VPC_ID}" "Name=association.main,Values=true" \
   --query "RouteTables[*].RouteTableId" \
   --region ${AURORA_REGION} \
   --output text

diff --git a/provision/aws/rds/aurora_delete.sh b/provision/aws/rds/aurora_delete.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+set -e
 
 if [[ "$RUNNER_DEBUG" == "1" ]]; then
   set -x
@@ -8,56 +9,60 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 source ${SCRIPT_DIR}/aurora_common.sh
 
 AURORA_VPC=$(aws ec2 describe-vpcs \
-  --filters "Name=cidr-block,Values=${AURORA_VPC_CIDR}" "Name=tag:AuroraCluster,Values=${AURORA_CLUSTER}" \
+  --filters "Name=tag:AuroraCluster,Values=${AURORA_CLUSTER}" \
   --query "Vpcs[*].VpcId" \
+  --region ${AURORA_REGION} \
   --output text
 )
 
 # Delete the Aurora DB cluster and instances
-for i in $( aws rds describe-db-clusters --output json | jq -r .DBClusters[0].DBClusterMembers[].DBInstanceIdentifier ); do
+for i in $( aws rds describe-db-clusters --region ${AURORA_REGION} --output json | jq -r .DBClusters[0].DBClusterMembers[].DBInstanceIdentifier ); do
   echo "Deleting Aurora DB instance ${i}"
-  aws rds delete-db-instance --db-instance-identifier "${i}" --skip-final-snapshot || true
+  aws rds delete-db-instance --db-instance-identifier "${i}" --skip-final-snapshot --region ${AURORA_REGION} || true
 done
 
 aws rds delete-db-cluster \
   --db-cluster-identifier ${AURORA_CLUSTER} \
+  --region ${AURORA_REGION} \
   --skip-final-snapshot \
   || true
 
-for i in $( aws rds describe-db-clusters --output json | jq -r .DBClusters[0].DBClusterMembers[].DBInstanceIdentifier ); do
-  aws rds wait db-instance-deleted --db-instance-identifier "${i}"
+for i in $( aws rds describe-db-clusters --output json --region ${AURORA_REGION} | jq -r .DBClusters[0].DBClusterMembers[].DBInstanceIdentifier ); do
+  aws rds wait db-instance-deleted --db-instance-identifier --region ${AURORA_REGION} "${i}"
 done
 
-aws rds wait db-cluster-deleted --db-cluster-identifier ${AURORA_CLUSTER}
+aws rds wait db-cluster-deleted --db-cluster-identifier ${AURORA_CLUSTER} --region ${AURORA_REGION}
 
 # Delete the Aurora subnet group
-aws rds delete-db-subnet-group --db-subnet-group-name ${AURORA_SUBNET_GROUP_NAME} || true
+aws rds delete-db-subnet-group --db-subnet-group-name ${AURORA_SUBNET_GROUP_NAME} --region ${AURORA_REGION} || true
 
 # Delete the Aurora subnets
 AURORA_SUBNETS=$(aws ec2 describe-subnets \
   --filters "Name=vpc-id,Values=${AURORA_VPC}" \
   --query "Subnets[*].SubnetId" \
+  --region ${AURORA_REGION} \
   --output text
 )
 for AURORA_SUBNET in ${AURORA_SUBNETS}; do
-  aws ec2 delete-subnet --subnet-id ${AURORA_SUBNET}
+  aws ec2 delete-subnet --region ${AURORA_REGION} --subnet-id ${AURORA_SUBNET}
 done
 
 # Delete the Aurora VPC Security Group
 AURORA_SECURITY_GROUP_ID=$(aws ec2 describe-security-groups \
   --filters "Name=vpc-id,Values=${AURORA_VPC}" "Name=group-name,Values=${AURORA_SECURITY_GROUP_NAME}" \
   --query "SecurityGroups[*].GroupId" \
+  --region ${AURORA_REGION} \
   --output text
 )
 if [ -n "${AURORA_SECURITY_GROUP_ID}" ]; then
-  aws ec2 delete-security-group --group-id ${AURORA_SECURITY_GROUP_ID}
+  aws ec2 delete-security-group --group-id ${AURORA_SECURITY_GROUP_ID} --region ${AURORA_REGION}
 fi
 
 # Delete the Aurora VPC, retrying 5 times in case that dependencies are not removed instantly
 n=0
 until [ "$n" -ge 20 ]
 do
-   aws ec2 delete-vpc --vpc-id ${AURORA_VPC} && break
+   aws ec2 delete-vpc --vpc-id ${AURORA_VPC} --region ${AURORA_REGION} && break
    n=$((n+1))
    echo "Unable to remove VPC ${AURORA_VPC}. Attempt ${n}"
    sleep 10

diff --git a/provision/aws/rds/aurora_delete_global_db.sh b/provision/aws/rds/aurora_delete_global_db.sh
@@ -34,7 +34,6 @@ for (( i = ${#GLOBAL_REGIONS[@]} - 1 ; i >= 0 ; i-- )) ; do
   REGION=${GLOBAL_REGIONS[i]}
   export AURORA_CLUSTER=${AURORA_GLOBAL_CLUSTER}-${REGION}
   export AURORA_REGION=${REGION}
-  export AURORA_VPC_CIDR=$(globalAuroraVpcCidr $i)
 
   AURORA_CLUSTER_ARN=$(aws rds describe-db-clusters \
     --query "DBClusters[?DBClusterIdentifier=='${AURORA_CLUSTER}'].DBClusterArn" \

diff --git a/provision/aws/rds/aurora_delete_global_peering_connections.sh b/provision/aws/rds/aurora_delete_global_peering_connections.sh
@@ -19,6 +19,5 @@ for (( i = 0 ; i < ${#GLOBAL_REGIONS[@]} ; i++ )) ; do
   REGION=${GLOBAL_REGIONS[i]}
   export AURORA_CLUSTER=${AURORA_GLOBAL_CLUSTER}-${REGION}
   export AURORA_REGION=${REGION}
-  export AURORA_VPC_CIDR=$(globalAuroraVpcCidr $i)
   ${SCRIPT_DIR}/aurora_delete_peering_connection.sh || true
 done
diff --git a/provision/aws/rds/aurora_delete_peering_connection.sh b/provision/aws/rds/aurora_delete_peering_connection.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+set -e
 
 if [[ "$RUNNER_DEBUG" == "1" ]]; then
   set -x
@@ -44,49 +45,49 @@ ROSA_PUBLIC_ROUTE_TABLE_ID=$(aws ec2 describe-route-tables \
   --output text
 )
 
+AURORA_VPC=$(aws ec2 describe-vpcs \
+  --filters "Name=tag:AuroraCluster,Values=${AURORA_CLUSTER}" \
+  --query 'Vpcs[0]' \
+  --output json \
+  --region ${AURORA_REGION}
+)
+AURORA_VPC_ID=$(echo ${AURORA_VPC} | jq -r .VpcId)
+AURORA_VPC_CIDR=$(echo ${AURORA_VPC} | jq -r .CidrBlock)
+
 if [ -n "${ROSA_PUBLIC_ROUTE_TABLE_ID}" ]; then
   aws ec2 delete-route \
     --route-table-id ${ROSA_PUBLIC_ROUTE_TABLE_ID} \
     --destination-cidr-block ${AURORA_VPC_CIDR} || true
 fi
 
 # Cleanup Aurora Region
-if [ -n ${AURORA_REGION} ]; then
-  AURORA_VPC=$(aws ec2 describe-vpcs \
-    --filters "Name=cidr,Values=${AURORA_VPC_CIDR}" "Name=tag:AuroraCluster,Values=${AURORA_CLUSTER}" \
-    --query 'Vpcs[*].VpcId' \
-    --region ${AURORA_REGION} \
-    --output text
-  )
+# Remove the ROSA route from the Aurora cluster
+AURORA_PUBLIC_ROUTE_TABLE_ID=$(aws ec2 describe-route-tables \
+  --filters "Name=vpc-id,Values=${AURORA_VPC_ID}" "Name=association.main,Values=true" \
+  --query "RouteTables[*].RouteTableId" \
+  --region ${AURORA_REGION} \
+  --output text
+)
 
-  # Remove the ROSA route from the Aurora cluster
-  AURORA_PUBLIC_ROUTE_TABLE_ID=$(aws ec2 describe-route-tables \
-    --filters "Name=vpc-id,Values=${AURORA_VPC}" "Name=association.main,Values=true" \
-    --query "RouteTables[*].RouteTableId" \
+if [ -n "${AURORA_PUBLIC_ROUTE_TABLE_ID}" ]; then
+  aws ec2 delete-route \
+    --route-table-id ${AURORA_PUBLIC_ROUTE_TABLE_ID} \
+    --destination-cidr-block ${ROSA_MACHINE_CIDR} \
     --region ${AURORA_REGION} \
-    --output text
-  )
-
-  if [ -n "${AURORA_PUBLIC_ROUTE_TABLE_ID}" ]; then
-    aws ec2 delete-route \
-      --route-table-id ${AURORA_PUBLIC_ROUTE_TABLE_ID} \
-      --destination-cidr-block ${ROSA_MACHINE_CIDR} \
-      --region ${AURORA_REGION} \
-      || true
-  fi
+    || true
+fi
 
-  AURORA_SECURITY_GROUP_ID=$(aws ec2 describe-security-groups \
-    --filters "Name=vpc-id,Values=${AURORA_VPC}" "Name=group-name,Values=${AURORA_SECURITY_GROUP_NAME}" \
-    --query "SecurityGroups[*].GroupId" \
-    --region ${AURORA_REGION} \
-    --output text
-  )
-  if [ -n "${AURORA_SECURITY_GROUP_ID}" ]; then
-    aws ec2 revoke-security-group-ingress \
-      --group-id ${AURORA_SECURITY_GROUP_ID} \
-      --protocol tcp \
-      --port 5432 \
-      --cidr ${ROSA_MACHINE_CIDR} \
-      --region ${AURORA_REGION}
-  fi
+AURORA_SECURITY_GROUP_ID=$(aws ec2 describe-security-groups \
+  --filters "Name=vpc-id,Values=${AURORA_VPC_ID}" "Name=group-name,Values=${AURORA_SECURITY_GROUP_NAME}" \
+  --query "SecurityGroups[*].GroupId" \
+  --region ${AURORA_REGION} \
+  --output text
+)
+if [ -n "${AURORA_SECURITY_GROUP_ID}" ]; then
+  aws ec2 revoke-security-group-ingress \
+    --group-id ${AURORA_SECURITY_GROUP_ID} \
+    --protocol tcp \
+    --port 5432 \
+    --cidr ${ROSA_MACHINE_CIDR} \
+    --region ${AURORA_REGION}
 fi
diff --git a/provision/aws/rds/aurora_global_common.sh b/provision/aws/rds/aurora_global_common.sh
@@ -23,15 +23,3 @@ function globalClusterRegions() {
   done
   echo ${AURORA_GLOBAL_REGIONS}
 }
-
-function globalAuroraVpcCidr() {
-  echo "192.168.$1.0/24"
-}
-
-function globalAuroraSubnetA() {
-  echo "192.168.$1.0/25"
-}
-
-function globalAuroraSubnetB() {
-  echo "192.168.$1.128/25"
-}