-
Notifications
You must be signed in to change notification settings - Fork 13
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
ci: add test workflow for docker-based replication (#254)
* ci: add test workflow for docker-based replication * doc: add notes to README
- Loading branch information
Showing
8 changed files
with
211 additions
and
44 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,140 @@ | ||
| name: Docker Replica Mode Test | ||
|
|
||
| on: | ||
| push: | ||
| branches: [ "main" ] | ||
| pull_request: | ||
| branches: [ "main" ] | ||
|
|
||
| jobs: | ||
| test-replication: | ||
| runs-on: ubuntu-latest | ||
| strategy: | ||
| matrix: | ||
| source: ['mysql', 'postgres'] | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
|
|
||
| - name: Install dependencies | ||
| run: | | ||
| # Only install DuckDB for data comparison | ||
| curl -LJO https://github.com/duckdb/duckdb/releases/latest/download/duckdb_cli-linux-amd64.zip | ||
| unzip duckdb_cli-linux-amd64.zip | ||
| chmod +x duckdb | ||
| sudo mv duckdb /usr/local/bin | ||
| - name: Start source ${{ matrix.source }} database | ||
| run: | | ||
| if [ "${{ matrix.source }}" = "mysql" ]; then | ||
| docker run -d --name source-db -p 3306:3306 \ | ||
| -e MYSQL_ROOT_PASSWORD=root \ | ||
| -e MYSQL_DATABASE=test \ | ||
| mysql:lts | ||
| # Wait for MySQL to be ready | ||
| until docker exec source-db mysql -uroot -proot -e "SELECT 1"; do | ||
| sleep 1 | ||
| done | ||
| # Create test data | ||
| docker exec source-db mysql -uroot -proot test -e " | ||
| CREATE TABLE items (id INT PRIMARY KEY, name VARCHAR(50)); | ||
| INSERT INTO items VALUES (1, 'test1'), (2, 'test2');" | ||
| else | ||
| docker run -d --name source-db -p 5432:5432 \ | ||
| -e POSTGRES_PASSWORD=postgres \ | ||
| -e POSTGRES_DB=test \ | ||
| postgres:latest \ | ||
| -c wal_level=logical | ||
| # Wait for PostgreSQL to be ready | ||
| until docker exec source-db pg_isready; do | ||
| sleep 1 | ||
| done | ||
| # Create test data | ||
| docker exec source-db psql -U postgres test -c " | ||
| CREATE TABLE items (id INT PRIMARY KEY, name VARCHAR(50)); | ||
| INSERT INTO items VALUES (1, 'test1'), (2, 'test2');" | ||
| fi | ||
| - name: Start MyDuck Server in replica mode | ||
| run: | | ||
| if [ "${{ matrix.source }}" = "mysql" ]; then | ||
| SOURCE_DSN="mysql://root:[email protected]:3306" | ||
| else | ||
| SOURCE_DSN="postgres://postgres:[email protected]:5432/test" | ||
| fi | ||
| docker run -d --name myduck \ | ||
| --add-host=host.docker.internal:host-gateway \ | ||
| -p 13306:3306 \ | ||
| -p 15432:5432 \ | ||
| --env=SETUP_MODE=REPLICA \ | ||
| --env=SOURCE_DSN="$SOURCE_DSN" \ | ||
| apecloud/myduckserver:latest | ||
| # Wait for MyDuck to be ready | ||
| sleep 10 | ||
| - name: Verify initial replication | ||
| run: | | ||
| # Query source data | ||
| if [ "${{ matrix.source }}" = "mysql" ]; then | ||
| docker exec source-db mysql -uroot -proot test \ | ||
| -e "SELECT * FROM items ORDER BY id;" > source_data.csv | ||
| else | ||
| docker exec source-db psql -U postgres -h 127.0.0.1 test \ | ||
| -c "\COPY (SELECT * FROM items ORDER BY id) TO STDOUT WITH CSV;" | tee source_data.csv | ||
| fi | ||
| # Query MyDuck data through Postgres interface | ||
| docker exec myduck psql -U postgres -h 127.0.0.1 \ | ||
| -c "\COPY (SELECT * FROM items ORDER BY id) TO STDOUT WITH CSV;" | tee myduck_data.csv | ||
| # Compare data using DuckDB | ||
| duckdb --csv -c " | ||
| CREATE TABLE source AS FROM 'source_data.csv'; | ||
| CREATE TABLE myduck AS FROM 'myduck_data.csv'; | ||
| SELECT COUNT(*) FROM ( | ||
| SELECT * FROM source EXCEPT SELECT * FROM myduck | ||
| ) diff;" | tail -n 1 | tee diff_count.txt | ||
| # Verify no differences | ||
| if grep -q '^0$' diff_count.txt; then | ||
| echo 'Initial replication verification successful' | ||
| else | ||
| echo 'Initial replication verification failed' | ||
| exit 1 | ||
| fi | ||
| - name: Test replication of new data | ||
| run: | | ||
| # Insert new data in source | ||
| if [ "${{ matrix.source }}" = "mysql" ]; then | ||
| docker exec source-db mysql -uroot -proot test \ | ||
| -e "INSERT INTO items VALUES (3, 'test3');" | ||
| else | ||
| docker exec source-db psql -U postgres test \ | ||
| -c "INSERT INTO items VALUES (3, 'test3');" | ||
| fi | ||
| # Wait for replication | ||
| sleep 5 | ||
| # Verify new data was replicated | ||
| docker exec myduck psql -t -U postgres -h 127.0.0.1 -c \ | ||
| "SELECT COUNT(*) FROM items WHERE id = 3;" | tr -d ' ' | tee count.txt | ||
| if grep -q '^1$' count.txt; then | ||
| echo 'Replication of new data verified successfully' | ||
| else | ||
| echo 'Replication of new data verification failed' | ||
| exit 1 | ||
| fi | ||
| - name: Cleanup | ||
| if: always() | ||
| run: | | ||
| docker rm -f source-db myduck || true |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -122,15 +122,17 @@ psql -h 127.0.0.1 -p 15432 -U postgres | |
|
|
||
| We have integrated a setup tool in the Docker image that helps replicate data from your primary (MySQL|Postgres) server to MyDuck Server. The tool is available via the `SETUP_MODE` environment variable. In `REPLICA` mode, the container will start MyDuck Server, dump a snapshot of your primary (MySQL|Postgres) server, and start replicating data in real-time. | ||
|
|
||
| > [!NOTE] | ||
| > Supported primary database versions: MySQL>=8.0 and PostgreSQL>=13. In addition to the default settings, | ||
| logical replication must be enabled for PostgreSQL by setting `wal_level=logical`. | ||
| > For MySQL, GTID-based replication (`gtid_mode=ON` and `enforce_gtid_consistency=ON`) is recommended but not required. | ||
| ```bash | ||
| docker run \ | ||
| docker run -d --name myduck \ | ||
| -p 13306:3306 \ | ||
| -p 15432:5432 \ | ||
| --privileged \ | ||
| --workdir=/home/admin \ | ||
| --env=SETUP_MODE=REPLICA \ | ||
| --env=SOURCE_DSN="<postgresql|mysql>://<user>:<password>@<host>:<port>/<dbname>" | ||
| --detach=true \ | ||
| apecloud/myduckserver:latest | ||
| ``` | ||
| `SOURCE_DSN` specifies the connection string to the primary database server, which can be either MySQL or PostgreSQL. | ||
|
|
@@ -141,6 +143,9 @@ docker run \ | |
| - **PostgreSQL Primary:** Use the `postgres` URI scheme, e.g., | ||
| `--env=SOURCE_DSN=postgres://postgres:[email protected]:5432` | ||
|
|
||
| > [!NOTE] | ||
| > To replicate from a server running on the host machine, use `host.docker.internal` as the hostname instead of `localhost` or `127.0.0.1`. On Linux, you must also add `--add-host=host.docker.internal:host-gateway` to the `docker run` command. | ||
| ### Connecting to Cloud MySQL & Postgres | ||
|
|
||
| MyDuck Server supports setting up replicas from common cloud-based MySQL & Postgres offerings. For more information, please refer to the [replica setup guide](docs/tutorial/replica-setup-rds.md). | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.