ci: add test workflow for docker-based replication

.github/workflows/replication-test.yml

@@ -0,0 +1,133 @@
+name: Replication Test
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  test-replication:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        source: ['mysql', 'postgres']
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          # Only install DuckDB for data comparison
+          curl -LJO https://github.com/duckdb/duckdb/releases/latest/download/duckdb_cli-linux-amd64.zip
+          unzip duckdb_cli-linux-amd64.zip
+          chmod +x duckdb
+          sudo mv duckdb /usr/local/bin
+
+      - name: Start source ${{ matrix.source }} database
+        run: |
+          if [ "${{ matrix.source }}" = "mysql" ]; then
+            docker run -d --name source-db -p 3306:3306 \
+              -e MYSQL_ROOT_PASSWORD=root \
+              -e MYSQL_DATABASE=test \
+              mysql:lts
+            # Wait for MySQL to be ready
+            until docker exec source-db mysql -uroot -proot -e "SELECT 1"; do
+              sleep 1
+            done
+            # Create test data
+            docker exec source-db mysql -uroot -proot test -e "
+              CREATE TABLE items (id INT PRIMARY KEY, name VARCHAR(50));
+              INSERT INTO items VALUES (1, 'test1'), (2, 'test2');"
+          else
+            docker run -d --name source-db -p 5432:5432 \
+              -e POSTGRES_PASSWORD=postgres \
+              -e POSTGRES_DB=test \
+              postgres:latest
+            # Wait for PostgreSQL to be ready
+            until docker exec source-db pg_isready; do
+              sleep 1
+            done
+            # Create test data
+            docker exec source-db psql -U postgres test -c "
+              CREATE TABLE items (id INT PRIMARY KEY, name VARCHAR(50));
+              INSERT INTO items VALUES (1, 'test1'), (2, 'test2');"
+          fi
+
+      - name: Start MyDuck Server in replica mode
+        run: |
+          if [ "${{ matrix.source }}" = "mysql" ]; then
+            SOURCE_DSN="mysql://root:[email protected]:3306/test"
+          else
+            SOURCE_DSN="postgres://postgres:[email protected]:5432/test"
+          fi
+
+          docker run -d --name myduck \
+            -p 13306:3306 \
+            -p 15432:5432 \
+            --env=SETUP_MODE=REPLICA \
+            --env=SOURCE_DSN="$SOURCE_DSN" \
+            apecloud/myduckserver:latest
+
+          # Wait for MyDuck to be ready
+          sleep 10
+
+      - name: Verify initial replication
+        run: |
+          # Query source data
+          if [ "${{ matrix.source }}" = "mysql" ]; then
+            docker exec source-db mysql -uroot -proot test \
+              -e "SELECT * FROM items ORDER BY id;" > source_data.csv
+          else
+            docker exec -i source-db psql -U postgres -h 127.0.0.1 test \
+              -c "\COPY (SELECT * FROM items ORDER BY id) TO STDOUT WITH CSV;" > source_data.csv
+          fi
+
+          # Query MyDuck data through Postgres interface
+          docker exec myduck psql -U postgres -h 127.0.0.1 \
+            -c "\COPY (SELECT * FROM items ORDER BY id) TO STDOUT WITH CSV;" > myduck_data.csv
+
+          # Compare data using DuckDB
+          duckdb --csv -c "
+            CREATE TABLE source AS FROM 'source_data.csv';
+            CREATE TABLE myduck AS FROM 'myduck_data.csv';
+            SELECT COUNT(*) FROM (
+              SELECT * FROM source EXCEPT SELECT * FROM myduck
+            ) diff;" | tail -n 1 | tee diff_count.txt
+
+          # Verify no differences
+          if grep -q '^0$' diff_count.txt; then
+            echo 'Initial replication verification successful'
+          else
+            echo 'Initial replication verification failed'
+            exit 1
+          fi
+
+      - name: Test replication of new data
+        run: |
+          # Insert new data in source
+          if [ "${{ matrix.source }}" = "mysql" ]; then
+            docker exec source-db mysql -uroot -proot test \
+              -e "INSERT INTO items VALUES (3, 'test3');"
+          else
+            docker exec source-db psql -U postgres test \
+              -c "INSERT INTO items VALUES (3, 'test3');"
+          fi
+
+          # Wait for replication
+          sleep 5
+
+          # Verify new data was replicated
+          docker exec -i myduck psql -U postgres -h 127.0.0.1 -t -c \
+            "SELECT COUNT(*) FROM items WHERE id = 3;" | tr -d ' ' | tee count.txt
+
+          if grep -q '^1$' count.txt; then
+            echo 'Replication of new data verified successfully'
+          else
+            echo 'Replication of new data verification failed'
+            exit 1
+          fi
+
+      - name: Cleanup
+        if: always()
+        run: |
+          docker rm -f source-db myduck || true

README.md

@@ -123,14 +123,11 @@ psql -h 127.0.0.1 -p 15432 -U postgres
 We have integrated a setup tool in the Docker image that helps replicate data from your primary (MySQL|Postgres) server to MyDuck Server. The tool is available via the `SETUP_MODE` environment variable. In `REPLICA` mode, the container will start MyDuck Server, dump a snapshot of your primary (MySQL|Postgres) server, and start replicating data in real-time.
 
 ```bash
-docker run \
+docker run -d --name myduck \
   -p 13306:3306 \ 
   -p 15432:5432 \
-  --privileged \
-  --workdir=/home/admin \
   --env=SETUP_MODE=REPLICA \
   --env=SOURCE_DSN="<postgresql|mysql>://<user>:<password>@<host>:<port>/<dbname>"
-  --detach=true \
   apecloud/myduckserver:latest
 ```
 `SOURCE_DSN` specifies the connection string to the primary database server, which can be either MySQL or PostgreSQL.