-
Notifications
You must be signed in to change notification settings - Fork 14
144 lines (123 loc) · 4.79 KB
/
replication-test.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
name: Docker Replica Mode Test
on:
push:
branches: [ "main" ]
jobs:
test-replication:
runs-on: ubuntu-latest
strategy:
matrix:
source: ['mysql', 'postgres']
steps:
- uses: actions/checkout@v4
- name: Install dependencies
run: |
# Only install DuckDB for data comparison
curl -LJO https://github.com/duckdb/duckdb/releases/latest/download/duckdb_cli-linux-amd64.zip
unzip duckdb_cli-linux-amd64.zip
chmod +x duckdb
sudo mv duckdb /usr/local/bin
- name: Start source ${{ matrix.source }} database
run: |
if [ "${{ matrix.source }}" = "mysql" ]; then
docker run -d --name source-db -p 3306:3306 \
-e MYSQL_ROOT_PASSWORD=root \
-e MYSQL_DATABASE=test \
mysql:lts
# Wait for MySQL to be ready
until docker exec source-db mysql -uroot -proot -e "SELECT 1"; do
sleep 1
done
# Create test data
docker exec source-db mysql -uroot -proot test -e "
CREATE TABLE items (id INT PRIMARY KEY, name VARCHAR(50));
INSERT INTO items VALUES (1, 'test1'), (2, 'test2');"
else
docker run -d --name source-db -p 5432:5432 \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=test \
postgres:latest \
-c wal_level=logical
# Wait for PostgreSQL to be ready
until docker exec source-db pg_isready; do
sleep 1
done
# Create test data
docker exec source-db psql -U postgres test -c "
CREATE TABLE items (id INT PRIMARY KEY, name VARCHAR(50));
INSERT INTO items VALUES (1, 'test1'), (2, 'test2');"
fi
- name: Start MyDuck Server in replica mode
run: |
if [ "${{ matrix.source }}" = "mysql" ]; then
SOURCE_DSN="mysql://root:[email protected]:3306"
else
SOURCE_DSN="postgres://postgres:[email protected]:5432/test"
fi
docker run -d --name myduck \
--add-host=host.docker.internal:host-gateway \
-p 13306:3306 \
-p 15432:5432 \
--env=SETUP_MODE=REPLICA \
--env=SOURCE_DSN="$SOURCE_DSN" \
apecloud/myduckserver:latest
# Wait for MyDuck to be ready
sleep 10
- name: Verify initial replication
run: |
# Query source data
SCHEMA=""
if [ "${{ matrix.source }}" = "mysql" ]; then
docker exec source-db mysql -uroot -proot test \
-e "SELECT * FROM items ORDER BY id;" | tee source_data.csv
SCHEMA=test
else
docker exec source-db psql -U postgres -h 127.0.0.1 test \
-c "\COPY (SELECT * FROM items ORDER BY id) TO STDOUT WITH CSV;" | tee source_data.csv
SCHEMA=public
fi
# Query MyDuck data
docker exec myduck psql -U postgres -h 127.0.0.1 \
-c "\COPY (SELECT * FROM ${SCHEMA}.items ORDER BY id) TO STDOUT WITH CSV;" | tee myduck_data.csv
# Compare data using DuckDB
duckdb --csv -c "
CREATE TABLE source AS FROM 'source_data.csv';
CREATE TABLE myduck AS FROM 'myduck_data.csv';
SELECT COUNT(*) FROM (
SELECT * FROM source EXCEPT SELECT * FROM myduck
) diff;" | tail -n 1 | tee diff_count.txt
# Verify no differences
if grep -q '^0$' diff_count.txt; then
echo 'Initial replication verification successful'
else
echo 'Initial replication verification failed'
exit 1
fi
- name: Test replication of new data
run: |
# Insert new data in source
SCHEMA=""
if [ "${{ matrix.source }}" = "mysql" ]; then
docker exec source-db mysql -uroot -proot test \
-e "INSERT INTO items VALUES (3, 'test3');"
SCHEMA=test
else
docker exec source-db psql -U postgres test \
-c "INSERT INTO items VALUES (3, 'test3');"
SCHEMA=public
fi
# Wait for replication
sleep 5
# Verify new data was replicated
docker exec myduck psql -t -U postgres -h 127.0.0.1 -c \
"SELECT COUNT(*) FROM ${SCHEMA}.items WHERE id = 3;" | tr -d ' ' | tee count.txt
if grep -q '^1$' count.txt; then
echo 'Replication of new data verified successfully'
else
echo 'Replication of new data verification failed'
exit 1
fi
- name: Cleanup
if: always()
run: |
docker rm -f source-db myduck || true