INCIDENT-001: postgres doesnt have Point of Recoverability so it migh…

…t fail when there's no replicas left

Also upgrade cnpg version while I'm at it. We backup now to an R2 bucket that has the postgres backups.

services/databases/postgresql/cnpg-backup-secrets.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cnpg-backup-secret
+  namespace: pg
+type: Opaque
+stringData:
+  ACCESS_KEY_ID: <FILL-IN>
+  ACCESS_SECRET_KEY: <FILL-IN>

services/databases/postgresql/cnpg-cluster.yaml

@@ -3,6 +3,7 @@ apiVersion: postgresql.cnpg.io/v1
 kind: Cluster
 metadata:
   name: cnpg-cluster
+  namespace: pg
 spec:
   instances: 3
 
@@ -36,4 +37,34 @@ spec:
 
   storage:
     size: 10Gi
-    storageClass: longhorn-strict-local-retain
+    #backups are handled by cloudnative postgres
+    storageClass: longhorn-strict-local-no-backup
+
+  backup:
+    barmanObjectStore:
+      destinationPath: s3://niployments-postgres-backup/
+      endpointURL: https://52d22ed664e31a094229250acd87ccfb.eu.r2.cloudflarestorage.com
+      s3Credentials:
+        accessKeyId:
+          name: cnpg-backup-secret
+          key: ACCESS_KEY_ID
+        secretAccessKey:
+          name: cnpg-backup-secret
+          key: ACCESS_SECRET_KEY
+      wal:
+        compression: gzip
+    retentionPolicy: "15d"
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: cluster-backup-object-store
+  namespace: pg
+spec:
+  cluster:
+    name: cnpg-cluster
+  method: barmanObjectStore
+  #Run on sundays, tuesdays and thursdays
+  schedule: '0 0 0 * * 0,2,4'
+  backupOwnerReference: cluster
+  immediate: true

services/databases/postgresql/cnpg-secrets.yaml

@@ -6,6 +6,7 @@ stringData:
 kind: Secret
 metadata:
   name: tts-secret
+  namespace: pg
 type: kubernetes.io/basic-auth
 ---
 apiVersion: v1
@@ -15,6 +16,7 @@ stringData:
 kind: Secret
 metadata:
   name: ni-secret
+  namespace: pg
 type: kubernetes.io/basic-auth
 ---
 apiVersion: v1
@@ -24,6 +26,7 @@ stringData:
 kind: Secret
 metadata:
   name: plausible-secret
+  namespace: pg
 type: kubernetes.io/basic-auth
 ---
 apiVersion: v1
@@ -33,4 +36,5 @@ stringData:
 kind: Secret
 metadata:
   name: sinf-website-2023-secret
+  namespace: pg
 type: kubernetes.io/basic-auth

services/databases/postgresql/deploy-cnpg-dev.sh

@@ -7,10 +7,11 @@ port=5432  # Define the desired port here
 cnpg_dir='./services/databases/postgresql'
 pods=$(cat $cnpg_dir/cnpg-cluster.yaml | awk '{if ($1 == "instances:") print $2}')
 
-kubectl apply --server-side -f https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-1.22/releases/cnpg-1.22.2.yaml
+kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-1.23/releases/cnpg-1.23.2.yaml
 kubectl wait --for=condition=available=true -n cnpg-system deployment/cnpg-controller-manager --timeout=120s
 
 kubectl create namespace pg
+kubectl apply -f $(dirname $0)/cnpg-backup-secrets.yaml -n pg
 kubectl apply -f $(dirname $0)/cnpg-secrets.yaml -n pg
 kubectl apply -f $(dirname $0)/cnpg-cluster.yaml -n pg
 sleep 5  # Wait a little bit for first pod to be created

services/databases/postgresql/deploy-cnpg-prod.sh

@@ -4,10 +4,13 @@
 
 pods=$(cat $(dirname $0)/cnpg-cluster.yaml | awk '{if ($1 == "instances:") print $2}')
 
-kubectl apply --server-side -f https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-1.22/releases/cnpg-1.22.2.yaml
+# NOTE(luisd): https://cloudnative-pg.io/documentation/1.23/installation_upgrade/#server-side-apply-of-manifests 
+#  they recommend force conflicts because such errors might happend when upgrading the controler
+kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-1.23/releases/cnpg-1.23.2.yaml
 kubectl wait --for=condition=available=true -n cnpg-system deployment/cnpg-controller-manager --timeout=120s
 
 kubectl create namespace pg
+kubectl apply -f $(dirname $0)/cnpg-backup-secrets.yaml -n pg
 kubectl apply -f $(dirname $0)/cnpg-secrets.yaml -n pg
 kubectl apply -f $(dirname $0)/cnpg-cluster.yaml -n pg
 sleep 5  # Wait a little bit for first pod to be created

services/storage/longhorn/storageClasses/longhorn-strict-local-no-backup.yaml

@@ -0,0 +1,17 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: longhorn-strict-local-no-backup
+provisioner: driver.longhorn.io
+allowVolumeExpansion: true
+reclaimPolicy: "Delete"
+volumeBindingMode: Immediate
+parameters:
+  numberOfReplicas: "1"
+  staleReplicaTimeout: "720"
+  fromBackup: ""
+  fsType: "ext4"
+  dataLocality: "strict-local"
+  replicaAutoBalance: "ignored"
+#  diskSelector: "ssd,fast"
+#  nodeSelector: "storage,fast"