Postgres 17 Failover Slots (#31)

README.md

@@ -79,7 +79,9 @@ _Decodable provides a managed PyFlink service. Learn more [here](https://docs.de
 | [MSSQL CDC](mssql_cdc/)                                          | Enabling MSSQL in Docker with CDC, reading from it with Debezium, writing change events into AWS Kinesis             |
 | [Oracle CDC](oracle_cdc/)                                        | Configuring Oracle AWS RDS with LogMiner, reading from it with Debezium, writing change events into AWS Kinesis      | 
 | [DynamoDb CDC](dynamodb_cdc/)                                    | Configure DynamoDB to send change data to Kinesis, reading changes into Decodable for transformation or replication. |
-| [ Logical Decoding Message Examples](postgres-logical-decoding)  | How to retrieve logical decoding messages from the Postgres WAL                                                      |
+| [Logical Decoding Message Examples](postgres-logical-decoding)   | How to retrieve logical decoding messages from the Postgres WAL                                                      |
+| [Logical Replication on Postgres 16 Stand-By Servers](postgres-logical-replication-standby) | How to use logical replication on Postgres 16 stand-by servers                            |
+| [Postgres 17 Fail-Over Slots](failover-slots)                    | How to use fail-over slots with Postgres 1                                                                           |
 
 ### Flink SQL
 
@@ -96,4 +98,4 @@ _Decodable provides a managed PyFlink service. Learn more [here](https://docs.de
 
 ## License
 
-This code base is available under the Apache License, version 2.
+This code base is available under the Apache License, version 2.

failover-slots/00_init.sql

@@ -0,0 +1,29 @@
+BEGIN;
+
+create user replicator with replication encrypted password 'zufsob-kuvtum-bImxa6';
+SELECT pg_create_physical_replication_slot('replication_slot');
+
+CREATE SCHEMA inventory;
+
+-- customers
+CREATE TABLE inventory.customers (
+  id SERIAL NOT NULL PRIMARY KEY,
+  first_name VARCHAR(255) NOT NULL,
+  last_name VARCHAR(255) NOT NULL,
+  email VARCHAR(255) NOT NULL UNIQUE,
+  is_test_account BOOLEAN NOT NULL
+);
+
+ALTER SEQUENCE inventory.customers_id_seq RESTART WITH 1001;
+ALTER TABLE inventory.customers REPLICA IDENTITY FULL;
+
+INSERT INTO inventory.customers
+VALUES (default, 'Sally', 'Thomas', '[email protected]', FALSE),
+       (default, 'George', 'Bailey', '[email protected]', FALSE),
+       (default, 'Edward', 'Walker', '[email protected]', FALSE),
+       (default, 'Aidan', 'Barrett', '[email protected]', TRUE),
+       (default, 'Anne', 'Kretchmar', '[email protected]', TRUE),
+       (default, 'Melissa', 'Cole', '[email protected]', FALSE),
+       (default, 'Rosalie', 'Stewart', '[email protected]', FALSE);
+
+COMMIT;

failover-slots/README.md

@@ -0,0 +1,24 @@
+# Failover Slots with Postgres 17
+
+This example shows the usage of Postgres 17 failover replication slots with Postgres SQL-based interface to logical replication as well as with Decodable.
+It accompanies the blog post <TODO>.
+
+To run this example, you’ll need to have these things:
+
+* A free Decodable account
+* The Decodable CLI installed on your machine
+* A free ngrok account
+
+Retrieve your auth token from the ngrok web UI and put it into a file _.env_ like so:
+
+```
+NGROK_AUTHTOKEN=<your token>
+```
+
+Start everything by running:
+
+```
+docker compose up
+```
+
+Then, follow the instructions from the blog post.

failover-slots/decodable-resources.yaml

@@ -0,0 +1,66 @@
+---
+kind: secret
+metadata:
+    name: gm-inventorydb-password
+    tags:
+        context: failover-slots-demo
+spec_version: v1
+spec:
+    value_literal: "top-secret"
+---
+kind: connection
+metadata:
+    name: gm_inventorydb-source
+    tags:
+        context: failover-slots-demo
+spec_version: v2
+spec:
+    connector: postgres-cdc
+    type: source
+    stream_mappings:
+        - stream_name: gm_inventorydb__inventory__customers
+          external_resource_specifier:
+            database-name: inventorydb
+            schema-name: inventory
+            table-name: customers
+    properties:
+        hostname: "%DB_HOST%"
+        password: gm-inventorydb-password
+        port: "%DB_PORT%"
+        database-name: inventorydb
+        decoding.plugin.name: pgoutput
+        username: user
+---
+kind: stream
+metadata:
+    name: gm_inventorydb__inventory__customers
+    description: Automatically created stream for inventorydb.inventory.customers
+    tags:
+        context: failover-slots-demo
+spec_version: v1
+spec:
+    schema_v2:
+        fields:
+            - kind: physical
+              name: id
+              type: INT NOT NULL
+            - kind: physical
+              name: first_name
+              type: VARCHAR(255)
+            - kind: physical
+              name: last_name
+              type: VARCHAR(255)
+            - kind: physical
+              name: email
+              type: VARCHAR(255)
+            - kind: physical
+              name: is_test_account
+              type: BOOLEAN
+        constraints:
+            primary_key:
+                - id
+    type: CHANGE
+    properties:
+        partition.count: "2"
+        compaction.enable: "false"
+        properties.compression.type: zstd

failover-slots/docker-compose.yaml

@@ -0,0 +1,76 @@
+x-postgres-common:
+  &postgres-common
+  image: postgres:17-alpine
+  user: postgres
+  restart: always
+  healthcheck:
+    test: 'pg_isready -U user --dbname=inventorydb'
+    interval: 10s
+    timeout: 5s
+    retries: 5
+
+services:
+  ngrok:
+    image: ngrok/ngrok:latest
+    container_name: ngrok
+    # Sign up for an ngrok account at https://dashboard.ngrok.com/signup
+    # Get your auth-token from https://dashboard.ngrok.com/get-started/your-authtoken
+    # and put an .env file in the same folder as this Docker Compose file in the form
+    # NGROK_AUTHTOKEN=your_token_value
+    command: "start --all --config /etc/ngrok.yml"
+    volumes:
+      - ./ngrok.yml:/etc/ngrok.yml
+    ports:
+      - 4040:4040 # Web dashboard for ngrok
+    env_file: ".env"
+  pgbouncer:
+    image: edoburu/pgbouncer:latest
+    environment:
+      - DB_HOST=postgres_primary
+      - DB_PORT=5432
+      - DB_USER=user
+      - DB_PASSWORD=top-secret
+      - ADMIN_USERS=postgres,admin
+      - AUTH_TYPE=scram-sha-256
+    ports:
+      - 15432:5432
+  postgres_primary:
+    <<: *postgres-common
+    ports:
+      - 5432:5432
+    environment:
+      POSTGRES_USER: user
+      POSTGRES_DB: inventorydb
+      POSTGRES_PASSWORD: top-secret
+      POSTGRES_HOST_AUTH_METHOD: "scram-sha-256\nhost replication all 0.0.0.0/0 md5"
+      POSTGRES_INITDB_ARGS: "--auth-host=scram-sha-256"
+    command: |
+      postgres 
+      -c wal_level=logical 
+      -c hot_standby=on 
+      -c max_wal_senders=10 
+      -c max_replication_slots=10
+      -c synchronized_standby_slots=replication_slot
+    volumes:
+      - ./00_init.sql:/docker-entrypoint-initdb.d/00_init.sql
+
+  postgres_replica:
+    <<: *postgres-common
+    ports:
+      - 5433:5432
+    environment:
+      PGUSER: replicator
+      PGPASSWORD: zufsob-kuvtum-bImxa6
+    command: |
+      bash -c "
+      until pg_basebackup --pgdata=/var/lib/postgresql/data -R --slot=replication_slot --host=postgres_primary --port=5432
+      do
+      echo 'Waiting for primary to connect...'
+      sleep 1s
+      done
+      echo 'Backup done, starting replica...'
+      chmod 0700 /var/lib/postgresql/data
+      postgres -c wal_level=logical -c hot_standby=on -c hot_standby_feedback=1
+      "
+    depends_on:
+      - postgres_primary

failover-slots/ngrok.yml

@@ -0,0 +1,5 @@
+version: "2"
+tunnels:
+  pgbouncer:
+    addr: pgbouncer:5432
+    proto: tcp

postgres-logical-replication-standby/README.md

@@ -1,3 +1,3 @@
 # Postgres Logical Replication From Stand-Bys
 
-This project accompanies blog post <TODO>.
+This project accompanies the blog post [Using Stand-by Servers for Postgres Logical Replication](https://www.decodable.co/blog/postgres-logical-replication).