Merge pull request #72 from NIAEFEUP/fix/postgres-backups

INCIDENT-001: postgres doesnt have Point of Recoverability so it might fail when there's no replicas left

services/databases/postgresql/cnpg-backup-secrets.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cnpg-backup-secret
+  namespace: pg
+type: Opaque
+stringData:
+  ACCESS_KEY_ID: <FILL-IN>
+  ACCESS_SECRET_KEY: <FILL-IN>

services/databases/postgresql/cnpg-cluster.yaml

@@ -3,6 +3,7 @@ apiVersion: postgresql.cnpg.io/v1
 kind: Cluster
 metadata:
   name: cnpg-cluster
+  namespace: pg
 spec:
   instances: 3
 
@@ -35,5 +36,39 @@ spec:
           name: sinf-website-2023-secret
 
   storage:
-    size: 10Gi
-    storageClass: longhorn-strict-local-retain
+    size: 20Gi
+    #backups are handled by cloudnative postgres
+    storageClass: longhorn-strict-local-no-backup
+
+  postgresql:
+    parameters:
+      max_slot_wal_keep_size: "10GB"
+
+  backup:
+    barmanObjectStore:
+      destinationPath: s3://niployments-postgres-backup/
+      endpointURL: https://52d22ed664e31a094229250acd87ccfb.eu.r2.cloudflarestorage.com
+      s3Credentials:
+        accessKeyId:
+          name: cnpg-backup-secret
+          key: ACCESS_KEY_ID
+        secretAccessKey:
+          name: cnpg-backup-secret
+          key: ACCESS_SECRET_KEY
+      wal:
+        compression: gzip
+    retentionPolicy: "15d"
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: cluster-backup-object-store
+  namespace: pg
+spec:
+  cluster:
+    name: cnpg-cluster
+  method: barmanObjectStore
+  #Run on sundays, tuesdays and thursdays
+  schedule: '0 0 0 * * 0,2,4'
+  backupOwnerReference: cluster
+  immediate: true

services/databases/postgresql/cnpg-secrets.yaml

@@ -6,6 +6,7 @@ stringData:
 kind: Secret
 metadata:
   name: tts-secret
+  namespace: pg
 type: kubernetes.io/basic-auth
 ---
 apiVersion: v1
@@ -15,6 +16,7 @@ stringData:
 kind: Secret
 metadata:
   name: ni-secret
+  namespace: pg
 type: kubernetes.io/basic-auth
 ---
 apiVersion: v1
@@ -24,6 +26,7 @@ stringData:
 kind: Secret
 metadata:
   name: plausible-secret
+  namespace: pg
 type: kubernetes.io/basic-auth
 ---
 apiVersion: v1
@@ -33,4 +36,5 @@ stringData:
 kind: Secret
 metadata:
   name: sinf-website-2023-secret
+  namespace: pg
 type: kubernetes.io/basic-auth

services/databases/postgresql/deploy-cnpg-dev.sh

@@ -7,10 +7,11 @@ port=5432  # Define the desired port here
 cnpg_dir='./services/databases/postgresql'
 pods=$(cat $cnpg_dir/cnpg-cluster.yaml | awk '{if ($1 == "instances:") print $2}')
 
-kubectl apply --server-side -f https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-1.22/releases/cnpg-1.22.2.yaml
+kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-1.23/releases/cnpg-1.23.2.yaml
 kubectl wait --for=condition=available=true -n cnpg-system deployment/cnpg-controller-manager --timeout=120s
 
 kubectl create namespace pg
+kubectl apply -f $(dirname $0)/cnpg-backup-secrets.yaml -n pg
 kubectl apply -f $(dirname $0)/cnpg-secrets.yaml -n pg
 kubectl apply -f $(dirname $0)/cnpg-cluster.yaml -n pg
 sleep 5  # Wait a little bit for first pod to be created

services/databases/postgresql/deploy-cnpg-prod.sh

@@ -4,10 +4,13 @@
 
 pods=$(cat $(dirname $0)/cnpg-cluster.yaml | awk '{if ($1 == "instances:") print $2}')
 
-kubectl apply --server-side -f https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-1.22/releases/cnpg-1.22.2.yaml
+# NOTE(luisd): https://cloudnative-pg.io/documentation/1.23/installation_upgrade/#server-side-apply-of-manifests 
+#  they recommend force conflicts because such errors might happend when upgrading the controler
+kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-1.23/releases/cnpg-1.23.2.yaml
 kubectl wait --for=condition=available=true -n cnpg-system deployment/cnpg-controller-manager --timeout=120s
 
 kubectl create namespace pg
+kubectl apply -f $(dirname $0)/cnpg-backup-secrets.yaml -n pg
 kubectl apply -f $(dirname $0)/cnpg-secrets.yaml -n pg
 kubectl apply -f $(dirname $0)/cnpg-cluster.yaml -n pg
 sleep 5  # Wait a little bit for first pod to be created

services/storage/longhorn/storageClasses/longhorn-strict-local-no-backup.yaml

@@ -0,0 +1,17 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: longhorn-strict-local-no-backup
+provisioner: driver.longhorn.io
+allowVolumeExpansion: true
+reclaimPolicy: "Delete"
+volumeBindingMode: Immediate
+parameters:
+  numberOfReplicas: "1"
+  staleReplicaTimeout: "720"
+  fromBackup: ""
+  fsType: "ext4"
+  dataLocality: "strict-local"
+  replicaAutoBalance: "ignored"
+#  diskSelector: "ssd,fast"
+#  nodeSelector: "storage,fast"