Skip to content

Commit aac3052

Browse files
soedirgosoedirgo
committed
chore(testinfra): initial ami test
1 parent 15b87cc commit aac3052

File tree

4 files changed

+312
-10
lines changed

4 files changed

+312
-10
lines changed

‎.github/workflows/testinfra.yml

+5-8
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@ jobs:
2525

2626
- name: Run aio integration tests
2727
run: |
28-
pip3 install docker pytest pytest-testinfra
28+
# TODO: use poetry for pkg mgmt
29+
pip3 install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest pytest-testinfra[paramiko,docker] requests
2930
pytest -vv testinfra/test_all_in_one.py
3031
3132
test-ami:
@@ -59,12 +60,6 @@ jobs:
5960
with:
6061
endpoint: builders
6162

62-
- uses: docker/login-action@v3
63-
with:
64-
registry: ghcr.io
65-
username: ${{ github.actor }}
66-
password: ${{ secrets.GITHUB_TOKEN }}
67-
6863
- uses: docker/build-push-action@v5
6964
with:
7065
load: true
@@ -125,7 +120,9 @@ jobs:
125120
126121
- name: Run tests
127122
run: |
128-
echo TODO
123+
# TODO: use poetry for pkg mgmt
124+
pip3 install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest pytest-testinfra[paramiko,docker] requests
125+
pytest -vv testinfra/test_ami.py
129126
130127
- name: Cleanup resources on build cancellation
131128
if: ${{ cancelled() }}

‎testinfra/README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
## Prerequisites
44

55
```sh
6-
pip3 install docker pytest pytest-testinfra requests
6+
pip3 install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest pytest-testinfra[paramiko,docker] requests
77
```
88

99
## Running locally

‎testinfra/test_all_in_one.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
# scope='session' uses the same container for all the tests;
2525
# scope='function' uses a new container per test function.
2626
@pytest.fixture(scope="session")
27-
def host(request):
27+
def host():
2828
# We build the image with the Docker CLI in path instead of using docker-py
2929
# (official Docker SDK for Python) because the latter doesn't use BuildKit,
3030
# so things like `ARG TARGETARCH` don't work:

‎testinfra/test_ami.py

+305
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,305 @@
1+
import base64
2+
import boto3
3+
import gzip
4+
import pytest
5+
import requests
6+
import testinfra
7+
from ec2instanceconnectcli.EC2InstanceConnectLogger import EC2InstanceConnectLogger
8+
from ec2instanceconnectcli.EC2InstanceConnectKey import EC2InstanceConnectKey
9+
from time import sleep
10+
11+
postgresql_schema_sql_content = """
12+
ALTER DATABASE postgres SET "app.settings.jwt_secret" TO 'my_jwt_secret_which_is_not_so_secret';
13+
ALTER DATABASE postgres SET "app.settings.jwt_exp" TO 3600;
14+
15+
ALTER USER supabase_admin WITH PASSWORD 'postgres';
16+
ALTER USER postgres WITH PASSWORD 'postgres';
17+
ALTER USER authenticator WITH PASSWORD 'postgres';
18+
ALTER USER pgbouncer WITH PASSWORD 'postgres';
19+
ALTER USER supabase_auth_admin WITH PASSWORD 'postgres';
20+
ALTER USER supabase_storage_admin WITH PASSWORD 'postgres';
21+
ALTER USER supabase_replication_admin WITH PASSWORD 'postgres';
22+
ALTER ROLE supabase_read_only_user WITH PASSWORD 'postgres';
23+
ALTER ROLE supabase_admin SET search_path TO "$user",public,auth,extensions;
24+
"""
25+
realtime_env_content = ""
26+
adminapi_yaml_content = """
27+
port: 8085
28+
host: 0.0.0.0
29+
ref: aaaaaaaaaaaaaaaaaaaa
30+
jwt_secret: my_jwt_secret_which_is_not_so_secret
31+
metric_collectors:
32+
- filesystem
33+
- meminfo
34+
- netdev
35+
- loadavg
36+
- cpu
37+
- diskstats
38+
- vmstat
39+
node_exporter_additional_args:
40+
- '--collector.filesystem.ignored-mount-points=^/(boot|sys|dev|run).*'
41+
- '--collector.netdev.device-exclude=lo'
42+
cert_path: /etc/ssl/adminapi/server.crt
43+
key_path: /etc/ssl/adminapi/server.key
44+
upstream_metrics_refresh_duration: 60s
45+
pgbouncer_endpoints:
46+
- 'postgres://pgbouncer:postgres@localhost:6543/pgbouncer'
47+
fail2ban_socket: /var/run/fail2ban/fail2ban.sock
48+
upstream_metrics_sources:
49+
-
50+
name: system
51+
url: 'https://localhost:8085/metrics'
52+
labels_to_attach: [{name: supabase_project_ref, value: aaaaaaaaaaaaaaaaaaaa}, {name: service_type, value: db}]
53+
skip_tls_verify: true
54+
-
55+
name: postgresql
56+
url: 'http://localhost:9187/metrics'
57+
labels_to_attach: [{name: supabase_project_ref, value: aaaaaaaaaaaaaaaaaaaa}, {name: service_type, value: postgresql}]
58+
-
59+
name: gotrue
60+
url: 'http://localhost:9122/metrics'
61+
labels_to_attach: [{name: supabase_project_ref, value: aaaaaaaaaaaaaaaaaaaa}, {name: service_type, value: gotrue}]
62+
monitoring:
63+
disk_usage:
64+
enabled: true
65+
firewall:
66+
enabled: true
67+
internal_ports:
68+
- 9187
69+
- 8085
70+
- 9122
71+
privileged_ports:
72+
- 22
73+
privileged_ports_allowlist:
74+
- 0.0.0.0/0
75+
filtered_ports:
76+
- 5432
77+
- 6543
78+
unfiltered_ports:
79+
- 80
80+
- 443
81+
managed_rules_file: /etc/nftables/supabase_managed.conf
82+
pg_egress_collect_path: /tmp/pg_egress_collect.txt
83+
aws_config:
84+
creds:
85+
enabled: false
86+
check_frequency: 1h
87+
refresh_buffer_duration: 6h
88+
"""
89+
pgsodium_root_key_content = (
90+
"0000000000000000000000000000000000000000000000000000000000000000"
91+
)
92+
postgrest_base_conf_content = """
93+
db-uri = "postgres://authenticator:postgres@localhost:5432/postgres?application_name=postgrest"
94+
db-schema = "public, storage, graphql_public"
95+
db-anon-role = "anon"
96+
jwt-secret = "my_jwt_secret_which_is_not_so_secret"
97+
role-claim-key = ".role"
98+
openapi-mode = "ignore-privileges"
99+
db-use-legacy-gucs = true
100+
admin-server-port = 3001
101+
server-host = "localhost"
102+
db-pool-acquisition-timeout = 10
103+
max-rows = 1000
104+
db-extra-search-path = "public, extensions"
105+
"""
106+
gotrue_env_content = """
107+
API_EXTERNAL_URL=http://localhost
108+
GOTRUE_API_HOST=0.0.0.0
109+
GOTRUE_SITE_URL=
110+
GOTRUE_DB_DRIVER=postgres
111+
GOTRUE_DB_DATABASE_URL=postgres://supabase_auth_admin@localhost/postgres?sslmode=disable
112+
GOTRUE_JWT_ADMIN_ROLES=supabase_admin,service_role
113+
GOTRUE_JWT_AUD=authenticated
114+
GOTRUE_JWT_SECRET=my_jwt_secret_which_is_not_so_secret
115+
"""
116+
walg_config_json_content = """
117+
{
118+
"AWS_REGION": "ap-southeast-1",
119+
"WALG_S3_PREFIX": "",
120+
"PGDATABASE": "postgres",
121+
"PGUSER": "supabase_admin",
122+
"PGPORT": 5432,
123+
"WALG_DELTA_MAX_STEPS": 6,
124+
"WALG_COMPRESSION_METHOD": "lz4"
125+
}
126+
"""
127+
anon_key = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImFhYWFhYWFhYWFhYWFhYWFhYWFhIiwicm9sZSI6ImFub24iLCJpYXQiOjE2OTYyMjQ5NjYsImV4cCI6MjAxMTgwMDk2Nn0.QW95aRPA-4QuLzuvaIeeoFKlJP9J2hvAIpJ3WJ6G5zo"
128+
service_role_key = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImFhYWFhYWFhYWFhYWFhYWFhYWFhIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImlhdCI6MTY5NjIyNDk2NiwiZXhwIjoyMDExODAwOTY2fQ.Om7yqv15gC3mLGitBmvFRB3M4IsLsX9fXzTQnFM7lu0"
129+
supabase_admin_key = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImFhYWFhYWFhYWFhYWFhYWFhYWFhIiwicm9sZSI6InN1cGFiYXNlX2FkbWluIiwiaWF0IjoxNjk2MjI0OTY2LCJleHAiOjIwMTE4MDA5NjZ9.jrD3j2rBWiIx0vhVZzd1CXFv7qkAP392nBMadvXxk1c"
130+
init_json_content = f"""
131+
{{
132+
"jwt_secret": "my_jwt_secret_which_is_not_so_secret",
133+
"project_ref": "aaaaaaaaaaaaaaaaaaaa",
134+
"logflare_api_key": "",
135+
"logflare_pitr_errors_source": "",
136+
"logflare_postgrest_source": "",
137+
"logflare_pgbouncer_source": "",
138+
"logflare_db_source": "",
139+
"logflare_gotrue_source": "",
140+
"anon_key": "{anon_key}",
141+
"service_key": "{service_role_key}",
142+
"supabase_admin_key": "{supabase_admin_key}",
143+
"common_name": "db.aaaaaaaaaaaaaaaaaaaa.supabase.red",
144+
"region": "ap-southeast-1",
145+
"init_database_only": false
146+
}}
147+
"""
148+
149+
150+
# scope='session' uses the same container for all the tests;
151+
# scope='function' uses a new container per test function.
152+
@pytest.fixture(scope="session")
153+
def host():
154+
ec2 = boto3.resource("ec2", region_name="ap-southeast-1")
155+
images = list(
156+
ec2.images.filter(
157+
Filters=[{"Name": "name", "Values": ["supabase-postgres-ci-ami-test"]}]
158+
)
159+
)
160+
assert len(images) == 1
161+
image = images[0]
162+
163+
def gzip_then_base64_encode(s: str) -> str:
164+
return base64.b64encode(gzip.compress(s.encode())).decode()
165+
166+
instance = list(
167+
ec2.create_instances(
168+
BlockDeviceMappings=[
169+
{
170+
"DeviceName": "/dev/sda1",
171+
"Ebs": {
172+
"VolumeSize": 8, # gb
173+
"Encrypted": True,
174+
"DeleteOnTermination": True,
175+
"VolumeType": "gp3",
176+
},
177+
},
178+
],
179+
MetadataOptions={
180+
"HttpTokens": "required",
181+
"HttpEndpoint": "enabled",
182+
},
183+
IamInstanceProfile={"Name": "pg-ap-southeast-1"},
184+
InstanceType="t4g.micro",
185+
MinCount=1,
186+
MaxCount=1,
187+
ImageId=image.id,
188+
SecurityGroups=[
189+
"supabase-postgres-security-group",
190+
"pgbouncer-security-group",
191+
],
192+
UserData=f"""#cloud-config
193+
hostname: db-aaaaaaaaaaaaaaaaaaaa
194+
write_files:
195+
- {{path: /etc/postgresql.schema.sql, content: {gzip_then_base64_encode(postgresql_schema_sql_content)}, permissions: '0600', encoding: gz+b64}}
196+
- {{path: /etc/realtime.env, content: {gzip_then_base64_encode(realtime_env_content)}, permissions: '0664', encoding: gz+b64}}
197+
- {{path: /etc/adminapi/adminapi.yaml, content: {gzip_then_base64_encode(adminapi_yaml_content)}, permissions: '0600', owner: 'adminapi:root', encoding: gz+b64}}
198+
- {{path: /etc/postgresql-custom/pgsodium_root.key, content: {gzip_then_base64_encode(pgsodium_root_key_content)}, permissions: '0600', owner: 'postgres:postgres', encoding: gz+b64}}
199+
- {{path: /etc/postgrest/base.conf, content: {gzip_then_base64_encode(postgrest_base_conf_content)}, permissions: '0664', encoding: gz+b64}}
200+
- {{path: /etc/gotrue.env, content: {gzip_then_base64_encode(gotrue_env_content)}, permissions: '0664', encoding: gz+b64}}
201+
- {{path: /etc/wal-g/config.json, content: {gzip_then_base64_encode(walg_config_json_content)}, permissions: '0664', owner: 'wal-g:wal-g', encoding: gz+b64}}
202+
- {{path: /tmp/init.json, content: {gzip_then_base64_encode(init_json_content)}, permissions: '0600', encoding: gz+b64}}
203+
runcmd:
204+
- 'sudo echo \"pgbouncer\" \"postgres\" >> /etc/pgbouncer/userlist.txt'
205+
- 'cd /tmp && aws s3 cp --region ap-southeast-1 s3://init-scripts-staging/project/init.sh .'
206+
- 'bash init.sh "staging"'
207+
- 'rm -rf /tmp/*'
208+
""",
209+
TagSpecifications=[
210+
{
211+
"ResourceType": "instance",
212+
"Tags": [{"Key": "Name", "Value": "ci-ami-test"}],
213+
}
214+
],
215+
)
216+
)[0]
217+
instance.wait_until_running()
218+
# instance doesn't have public ip yet
219+
instance = ec2.Instance(instance.id)
220+
221+
logger = EC2InstanceConnectLogger(debug=False)
222+
temp_key = EC2InstanceConnectKey(logger.get_logger())
223+
ec2ic = boto3.client("ec2-instance-connect", region_name="ap-southeast-1")
224+
response = ec2ic.send_ssh_public_key(
225+
InstanceId=instance.id,
226+
InstanceOSUser="ubuntu",
227+
SSHPublicKey=temp_key.get_pub_key(),
228+
)
229+
assert response["Success"]
230+
231+
host = testinfra.get_host(
232+
# paramiko is an ssh backend
233+
f"paramiko://ubuntu@{instance.public_ip_address}",
234+
ssh_identity_file=temp_key.get_priv_key_file(),
235+
)
236+
237+
def is_healthy(host) -> bool:
238+
cmd = host.run("pg_isready -U postgres")
239+
if cmd.failed is True:
240+
return False
241+
242+
cmd = host.run(f"curl http://localhost:8085/health -H 'apikey: {supabase_admin_key}'")
243+
if cmd.failed is True:
244+
return False
245+
246+
cmd = host.run("curl http://localhost:3001/ready")
247+
if cmd.failed is True:
248+
return False
249+
250+
cmd = host.run("curl http://localhost:8081/health")
251+
if cmd.failed is True:
252+
return False
253+
254+
cmd = host.run("sudo kong health")
255+
if cmd.failed is True:
256+
return False
257+
258+
cmd = host.run("printf \\\\0 > '/dev/tcp/localhost/6543'")
259+
if cmd.failed is True:
260+
return False
261+
262+
cmd = host.run("sudo fail2ban-client status")
263+
if cmd.failed is True:
264+
return False
265+
266+
return True
267+
268+
while True:
269+
if is_healthy(host):
270+
break
271+
sleep(1)
272+
273+
# return a testinfra connection to the instance
274+
yield host
275+
276+
# at the end of the test suite, destroy the instance
277+
instance.terminate()
278+
279+
280+
def test_postgrest_is_running(host):
281+
postgrest = host.service("postgrest")
282+
assert postgrest.is_running
283+
284+
285+
def test_postgrest_responds_to_requests(host):
286+
res = requests.get(
287+
f"http://{host.backend.get_hostname()}/rest/v1/",
288+
headers={
289+
"apikey": anon_key,
290+
"authorization": f"Bearer {anon_key}",
291+
},
292+
)
293+
assert res.ok
294+
295+
296+
def test_postgrest_can_connect_to_db(host):
297+
res = requests.get(
298+
f"http://{host.backend.get_hostname()}/rest/v1/buckets",
299+
headers={
300+
"apikey": service_role_key,
301+
"authorization": f"Bearer {service_role_key}",
302+
"accept-profile": "storage",
303+
},
304+
)
305+
assert res.ok

0 commit comments

Comments
 (0)