ci: add test workflow for docker-based replication #1
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Replication Test | |
| on: | |
| push: | |
| branches: [ "main" ] | |
| pull_request: | |
| branches: [ "main" ] | |
| jobs: | |
| test-replication: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| source: ['mysql', 'postgres'] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install dependencies | |
| run: | | |
| # Only install DuckDB for data comparison | |
| curl -LJO https://github.com/duckdb/duckdb/releases/latest/download/duckdb_cli-linux-amd64.zip | |
| unzip duckdb_cli-linux-amd64.zip | |
| chmod +x duckdb | |
| sudo mv duckdb /usr/local/bin | |
| - name: Start source ${{ matrix.source }} database | |
| run: | | |
| if [ "${{ matrix.source }}" = "mysql" ]; then | |
| docker run -d --name source-db -p 3306:3306 \ | |
| -e MYSQL_ROOT_PASSWORD=root \ | |
| -e MYSQL_DATABASE=test \ | |
| mysql:lts | |
| # Wait for MySQL to be ready | |
| until docker exec source-db mysql -uroot -proot -e "SELECT 1"; do | |
| sleep 1 | |
| done | |
| # Create test data | |
| docker exec source-db mysql -uroot -proot test -e " | |
| CREATE TABLE items (id INT PRIMARY KEY, name VARCHAR(50)); | |
| INSERT INTO items VALUES (1, 'test1'), (2, 'test2');" | |
| else | |
| docker run -d --name source-db -p 5432:5432 \ | |
| -e POSTGRES_PASSWORD=postgres \ | |
| -e POSTGRES_DB=test \ | |
| postgres:latest | |
| # Wait for PostgreSQL to be ready | |
| until docker exec source-db pg_isready; do | |
| sleep 1 | |
| done | |
| # Create test data | |
| docker exec source-db psql -U postgres test -c " | |
| CREATE TABLE items (id INT PRIMARY KEY, name VARCHAR(50)); | |
| INSERT INTO items VALUES (1, 'test1'), (2, 'test2');" | |
| fi | |
| - name: Start MyDuck Server in replica mode | |
| run: | | |
| if [ "${{ matrix.source }}" = "mysql" ]; then | |
| SOURCE_DSN="mysql://root:[email protected]:3306/test" | |
| else | |
| SOURCE_DSN="postgres://postgres:[email protected]:5432/test" | |
| fi | |
| docker run -d --name myduck \ | |
| -p 13306:3306 \ | |
| -p 15432:5432 \ | |
| --env=SETUP_MODE=REPLICA \ | |
| --env=SOURCE_DSN="$SOURCE_DSN" \ | |
| apecloud/myduckserver:latest | |
| # Wait for MyDuck to be ready | |
| sleep 10 | |
| - name: Verify initial replication | |
| run: | | |
| # Query source data | |
| if [ "${{ matrix.source }}" = "mysql" ]; then | |
| docker exec source-db mysql -uroot -proot test \ | |
| -e "SELECT * FROM items ORDER BY id;" > source_data.csv | |
| else | |
| docker exec -i source-db psql -U postgres -h 127.0.0.1 test \ | |
| -c "\COPY (SELECT * FROM items ORDER BY id) TO STDOUT WITH CSV;" > source_data.csv | |
| fi | |
| # Query MyDuck data through Postgres interface | |
| docker exec myduck psql -U postgres -h 127.0.0.1 \ | |
| -c "\COPY (SELECT * FROM items ORDER BY id) TO STDOUT WITH CSV;" > myduck_data.csv | |
| # Compare data using DuckDB | |
| duckdb --csv -c " | |
| CREATE TABLE source AS FROM 'source_data.csv'; | |
| CREATE TABLE myduck AS FROM 'myduck_data.csv'; | |
| SELECT COUNT(*) FROM ( | |
| SELECT * FROM source EXCEPT SELECT * FROM myduck | |
| ) diff;" | tail -n 1 | tee diff_count.txt | |
| # Verify no differences | |
| if grep -q '^0$' diff_count.txt; then | |
| echo 'Initial replication verification successful' | |
| else | |
| echo 'Initial replication verification failed' | |
| exit 1 | |
| fi | |
| - name: Test replication of new data | |
| run: | | |
| # Insert new data in source | |
| if [ "${{ matrix.source }}" = "mysql" ]; then | |
| docker exec source-db mysql -uroot -proot test \ | |
| -e "INSERT INTO items VALUES (3, 'test3');" | |
| else | |
| docker exec source-db psql -U postgres test \ | |
| -c "INSERT INTO items VALUES (3, 'test3');" | |
| fi | |
| # Wait for replication | |
| sleep 5 | |
| # Verify new data was replicated | |
| docker exec -i myduck psql -U postgres -h 127.0.0.1 -t -c \ | |
| "SELECT COUNT(*) FROM items WHERE id = 3;" | tr -d ' ' | tee count.txt | |
| if grep -q '^1$' count.txt; then | |
| echo 'Replication of new data verified successfully' | |
| else | |
| echo 'Replication of new data verification failed' | |
| exit 1 | |
| fi | |
| - name: Cleanup | |
| if: always() | |
| run: | | |
| docker rm -f source-db myduck || true |