diff --git a/.github/workflows/testinfra.yml b/.github/workflows/testinfra.yml index bea5c5254..989333dc7 100644 --- a/.github/workflows/testinfra.yml +++ b/.github/workflows/testinfra.yml @@ -5,7 +5,7 @@ on: workflow_dispatch: jobs: - build: + test-all-in-one: strategy: matrix: include: @@ -25,5 +25,110 @@ jobs: - name: Run aio integration tests run: | - pip3 install docker pytest pytest-testinfra + # TODO: use poetry for pkg mgmt + pip3 install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest pytest-testinfra[paramiko,docker] requests pytest -vv testinfra/test_all_in_one.py + + test-ami: + strategy: + matrix: + include: + - runner: arm-runner + arch: arm64 + ubuntu_release: focal + ubuntu_version: 20.04 + mcpu: neoverse-n1 + runs-on: ${{ matrix.runner }} + timeout-minutes: 150 + permissions: + contents: write + packages: write + id-token: write + + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + + - id: args + uses: mikefarah/yq@master + with: + cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml' + + - run: docker context create builders + + - uses: docker/setup-buildx-action@v3 + with: + endpoint: builders + + - uses: docker/build-push-action@v5 + with: + load: true + build-args: | + ${{ steps.args.outputs.result }} + target: extensions + tags: supabase/postgres:extensions + platforms: linux/${{ matrix.arch }} + cache-from: | + type=gha,scope=${{ github.ref_name }}-extensions + type=gha,scope=${{ github.base_ref }}-extensions + type=gha,scope=develop-extensions + cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-extensions + + - name: Extract built packages + run: | + mkdir -p /tmp/extensions ansible/files/extensions + docker save supabase/postgres:extensions | tar xv -C /tmp/extensions + for layer in /tmp/extensions/*/layer.tar; do + tar xvf "$layer" -C ansible/files/extensions --strip-components 1 + done + + - id: version + run: echo "${{ steps.args.outputs.result }}" | grep "postgresql" >> "$GITHUB_OUTPUT" + + - name: Build Postgres deb + uses: docker/build-push-action@v5 + with: + load: true + file: docker/Dockerfile + target: pg-deb + build-args: | + ubuntu_release=${{ matrix.ubuntu_release }} + ubuntu_release_no=${{ matrix.ubuntu_version }} + postgresql_major=${{ steps.version.outputs.postgresql_major }} + postgresql_release=${{ steps.version.outputs.postgresql_release }} + CPPFLAGS=-mcpu=${{ matrix.mcpu }} + tags: supabase/postgres:deb + platforms: linux/${{ matrix.arch }} + cache-from: | + type=gha,scope=${{ github.ref_name }}-deb + type=gha,scope=${{ github.base_ref }}-deb + type=gha,scope=develop-deb + cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-deb + + - name: Extract Postgres deb + run: | + mkdir -p /tmp/build ansible/files/postgres + docker save supabase/postgres:deb | tar xv -C /tmp/build + for layer in /tmp/build/*/layer.tar; do + tar xvf "$layer" -C ansible/files/postgres --strip-components 1 + done + + # Packer doesn't support skipping registering the AMI for the ebssurrogate + # builder, so we register an AMI with a fixed name and run tests on an + # instance launched from that + # https://github.com/hashicorp/packer/issues/4899 + - name: Build AMI + run: | + GIT_SHA=${{github.sha}} + packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common.vars.pkr.hcl" -var "ansible_arguments=" -var "postgres-version=ci-ami-test" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" amazon-arm64.pkr.hcl + + - name: Run tests + run: | + # TODO: use poetry for pkg mgmt + pip3 install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest pytest-testinfra[paramiko,docker] requests + pytest -vv testinfra/test_ami.py + + - name: Cleanup resources on build cancellation + if: ${{ cancelled() }} + run: | + aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -I {} aws ec2 terminate-instances --instance-ids {} diff --git a/amazon-arm64.pkr.hcl b/amazon-arm64.pkr.hcl index 1a10101b2..884a8944b 100644 --- a/amazon-arm64.pkr.hcl +++ b/amazon-arm64.pkr.hcl @@ -87,6 +87,11 @@ variable "packer-execution-id" { default = "unknown" } +variable "force-deregister" { + type = bool + default = false +} + # source block source "amazon-ebssurrogate" "source" { profile = "${var.profile}" @@ -99,6 +104,7 @@ source "amazon-ebssurrogate" "source" { instance_type = "c6g.4xlarge" region = "${var.region}" #secret_key = "${var.aws_secret_key}" + force_deregister = var.force-deregister # Use latest official ubuntu focal ami owned by Canonical. source_ami_filter { diff --git a/testinfra/README.md b/testinfra/README.md index 6477e2213..7c2dd86ed 100644 --- a/testinfra/README.md +++ b/testinfra/README.md @@ -2,14 +2,64 @@ ## Prerequisites +- Docker +- Packer +- yq +- Python deps: + ```sh -pip3 install docker pytest pytest-testinfra requests +pip3 install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest pytest-testinfra[paramiko,docker] requests ``` ## Running locally ```sh +set -euo pipefail # cwd: repo root # docker must be running -pytest -vv testinfra/*.py + +# build extensions & pg binaries +docker buildx build \ + $(yq 'to_entries | map(select(.value|type == "!!str")) | map(" --build-arg " + .key + "=" + .value) | join("")' 'ansible/vars.yml') \ + --target=extensions \ + --tag=supabase/postgres:extensions \ + --platform=linux/arm64 \ + --load \ + . +mkdir -p /tmp/extensions ansible/files/extensions +docker save supabase/postgres:extensions | tar xv -C /tmp/extensions +for layer in /tmp/extensions/*/layer.tar; do + tar xvf "$layer" -C ansible/files/extensions --strip-components 1 +done +docker buildx build \ + --build-arg ubuntu_release=focal \ + --build-arg ubuntu_release_no=20.04 \ + --build-arg postgresql_major=15 \ + --build-arg postgresql_release=15.1 \ + --build-arg CPPFLAGS=-mcpu=neoverse-n1 \ + --file=docker/Dockerfile \ + --target=pg-deb \ + --tag=supabase/postgres:deb \ + --platform=linux/arm64 \ + --load \ + . +mkdir -p /tmp/build ansible/files/postgres +docker save supabase/postgres:deb | tar xv -C /tmp/build +for layer in /tmp/build/*/layer.tar; do + tar xvf "$layer" -C ansible/files/postgres --strip-components 1 +done + +# build AMI +AWS_PROFILE=supabase-dev packer build \ + -var-file=development-arm.vars.pkr.hcl \ + -var-file=common.vars.pkr.hcl \ + -var "ansible_arguments=" \ + -var "postgres-version=ci-ami-test" \ + -var "region=ap-southeast-1" \ + -var 'ami_regions=["ap-southeast-1"]' \ + -var "force-deregister=true" \ + amazon-arm64.pkr.hcl + +# run tests +AWS_PROFILE=supabase-dev pytest -vv -s testinfra/test_*.py ``` diff --git a/testinfra/test_all_in_one.py b/testinfra/test_all_in_one.py index 7973e0faa..28a9f524f 100644 --- a/testinfra/test_all_in_one.py +++ b/testinfra/test_all_in_one.py @@ -1,4 +1,5 @@ from docker.models.containers import Container +from os import path from time import sleep from typing import cast import docker @@ -24,7 +25,7 @@ # scope='session' uses the same container for all the tests; # scope='function' uses a new container per test function. @pytest.fixture(scope="session") -def host(request): +def host(): # We build the image with the Docker CLI in path instead of using docker-py # (official Docker SDK for Python) because the latter doesn't use BuildKit, # so things like `ARG TARGETARCH` don't work: @@ -36,11 +37,11 @@ def host(request): "buildx", "build", "--file", - "docker/all-in-one/Dockerfile", + path.join(path.dirname(__file__), "../docker/all-in-one/Dockerfile"), "--load", "--tag", all_in_one_image_tag, - ".", + path.join(path.dirname(__file__), ".."), ] ) diff --git a/testinfra/test_ami.py b/testinfra/test_ami.py new file mode 100644 index 000000000..9b2c4486d --- /dev/null +++ b/testinfra/test_ami.py @@ -0,0 +1,307 @@ +import base64 +import boto3 +import gzip +import pytest +import requests +import testinfra +from ec2instanceconnectcli.EC2InstanceConnectLogger import EC2InstanceConnectLogger +from ec2instanceconnectcli.EC2InstanceConnectKey import EC2InstanceConnectKey +from time import sleep + +postgresql_schema_sql_content = """ +ALTER DATABASE postgres SET "app.settings.jwt_secret" TO 'my_jwt_secret_which_is_not_so_secret'; +ALTER DATABASE postgres SET "app.settings.jwt_exp" TO 3600; + +ALTER USER supabase_admin WITH PASSWORD 'postgres'; +ALTER USER postgres WITH PASSWORD 'postgres'; +ALTER USER authenticator WITH PASSWORD 'postgres'; +ALTER USER pgbouncer WITH PASSWORD 'postgres'; +ALTER USER supabase_auth_admin WITH PASSWORD 'postgres'; +ALTER USER supabase_storage_admin WITH PASSWORD 'postgres'; +ALTER USER supabase_replication_admin WITH PASSWORD 'postgres'; +ALTER ROLE supabase_read_only_user WITH PASSWORD 'postgres'; +ALTER ROLE supabase_admin SET search_path TO "$user",public,auth,extensions; +""" +realtime_env_content = "" +adminapi_yaml_content = """ +port: 8085 +host: 0.0.0.0 +ref: aaaaaaaaaaaaaaaaaaaa +jwt_secret: my_jwt_secret_which_is_not_so_secret +metric_collectors: + - filesystem + - meminfo + - netdev + - loadavg + - cpu + - diskstats + - vmstat +node_exporter_additional_args: + - '--collector.filesystem.ignored-mount-points=^/(boot|sys|dev|run).*' + - '--collector.netdev.device-exclude=lo' +cert_path: /etc/ssl/adminapi/server.crt +key_path: /etc/ssl/adminapi/server.key +upstream_metrics_refresh_duration: 60s +pgbouncer_endpoints: + - 'postgres://pgbouncer:postgres@localhost:6543/pgbouncer' +fail2ban_socket: /var/run/fail2ban/fail2ban.sock +upstream_metrics_sources: + - + name: system + url: 'https://localhost:8085/metrics' + labels_to_attach: [{name: supabase_project_ref, value: aaaaaaaaaaaaaaaaaaaa}, {name: service_type, value: db}] + skip_tls_verify: true + - + name: postgresql + url: 'http://localhost:9187/metrics' + labels_to_attach: [{name: supabase_project_ref, value: aaaaaaaaaaaaaaaaaaaa}, {name: service_type, value: postgresql}] + - + name: gotrue + url: 'http://localhost:9122/metrics' + labels_to_attach: [{name: supabase_project_ref, value: aaaaaaaaaaaaaaaaaaaa}, {name: service_type, value: gotrue}] +monitoring: + disk_usage: + enabled: true +firewall: + enabled: true + internal_ports: + - 9187 + - 8085 + - 9122 + privileged_ports: + - 22 + privileged_ports_allowlist: + - 0.0.0.0/0 + filtered_ports: + - 5432 + - 6543 + unfiltered_ports: + - 80 + - 443 + managed_rules_file: /etc/nftables/supabase_managed.conf +pg_egress_collect_path: /tmp/pg_egress_collect.txt +aws_config: + creds: + enabled: false + check_frequency: 1h + refresh_buffer_duration: 6h +""" +pgsodium_root_key_content = ( + "0000000000000000000000000000000000000000000000000000000000000000" +) +postgrest_base_conf_content = """ +db-uri = "postgres://authenticator:postgres@localhost:5432/postgres?application_name=postgrest" +db-schema = "public, storage, graphql_public" +db-anon-role = "anon" +jwt-secret = "my_jwt_secret_which_is_not_so_secret" +role-claim-key = ".role" +openapi-mode = "ignore-privileges" +db-use-legacy-gucs = true +admin-server-port = 3001 +server-host = "localhost" +db-pool-acquisition-timeout = 10 +max-rows = 1000 +db-extra-search-path = "public, extensions" +""" +gotrue_env_content = """ +API_EXTERNAL_URL=http://localhost +GOTRUE_API_HOST=0.0.0.0 +GOTRUE_SITE_URL= +GOTRUE_DB_DRIVER=postgres +GOTRUE_DB_DATABASE_URL=postgres://supabase_auth_admin@localhost/postgres?sslmode=disable +GOTRUE_JWT_ADMIN_ROLES=supabase_admin,service_role +GOTRUE_JWT_AUD=authenticated +GOTRUE_JWT_SECRET=my_jwt_secret_which_is_not_so_secret +""" +walg_config_json_content = """ +{ + "AWS_REGION": "ap-southeast-1", + "WALG_S3_PREFIX": "", + "PGDATABASE": "postgres", + "PGUSER": "supabase_admin", + "PGPORT": 5432, + "WALG_DELTA_MAX_STEPS": 6, + "WALG_COMPRESSION_METHOD": "lz4" +} +""" +anon_key = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImFhYWFhYWFhYWFhYWFhYWFhYWFhIiwicm9sZSI6ImFub24iLCJpYXQiOjE2OTYyMjQ5NjYsImV4cCI6MjAxMTgwMDk2Nn0.QW95aRPA-4QuLzuvaIeeoFKlJP9J2hvAIpJ3WJ6G5zo" +service_role_key = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImFhYWFhYWFhYWFhYWFhYWFhYWFhIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImlhdCI6MTY5NjIyNDk2NiwiZXhwIjoyMDExODAwOTY2fQ.Om7yqv15gC3mLGitBmvFRB3M4IsLsX9fXzTQnFM7lu0" +supabase_admin_key = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImFhYWFhYWFhYWFhYWFhYWFhYWFhIiwicm9sZSI6InN1cGFiYXNlX2FkbWluIiwiaWF0IjoxNjk2MjI0OTY2LCJleHAiOjIwMTE4MDA5NjZ9.jrD3j2rBWiIx0vhVZzd1CXFv7qkAP392nBMadvXxk1c" +init_json_content = f""" +{{ + "jwt_secret": "my_jwt_secret_which_is_not_so_secret", + "project_ref": "aaaaaaaaaaaaaaaaaaaa", + "logflare_api_key": "", + "logflare_pitr_errors_source": "", + "logflare_postgrest_source": "", + "logflare_pgbouncer_source": "", + "logflare_db_source": "", + "logflare_gotrue_source": "", + "anon_key": "{anon_key}", + "service_key": "{service_role_key}", + "supabase_admin_key": "{supabase_admin_key}", + "common_name": "db.aaaaaaaaaaaaaaaaaaaa.supabase.red", + "region": "ap-southeast-1", + "init_database_only": false +}} +""" + + +# scope='session' uses the same container for all the tests; +# scope='function' uses a new container per test function. +@pytest.fixture(scope="session") +def host(): + ec2 = boto3.resource("ec2", region_name="ap-southeast-1") + images = list( + ec2.images.filter( + Filters=[{"Name": "name", "Values": ["supabase-postgres-ci-ami-test"]}] + ) + ) + assert len(images) == 1 + image = images[0] + + def gzip_then_base64_encode(s: str) -> str: + return base64.b64encode(gzip.compress(s.encode())).decode() + + instance = list( + ec2.create_instances( + BlockDeviceMappings=[ + { + "DeviceName": "/dev/sda1", + "Ebs": { + "VolumeSize": 8, # gb + "Encrypted": True, + "DeleteOnTermination": True, + "VolumeType": "gp3", + }, + }, + ], + MetadataOptions={ + "HttpTokens": "required", + "HttpEndpoint": "enabled", + }, + IamInstanceProfile={"Name": "pg-ap-southeast-1"}, + InstanceType="t4g.micro", + MinCount=1, + MaxCount=1, + ImageId=image.id, + SecurityGroups=[ + "supabase-postgres-security-group", + "pgbouncer-security-group", + ], + UserData=f"""#cloud-config +hostname: db-aaaaaaaaaaaaaaaaaaaa +write_files: + - {{path: /etc/postgresql.schema.sql, content: {gzip_then_base64_encode(postgresql_schema_sql_content)}, permissions: '0600', encoding: gz+b64}} + - {{path: /etc/realtime.env, content: {gzip_then_base64_encode(realtime_env_content)}, permissions: '0664', encoding: gz+b64}} + - {{path: /etc/adminapi/adminapi.yaml, content: {gzip_then_base64_encode(adminapi_yaml_content)}, permissions: '0600', owner: 'adminapi:root', encoding: gz+b64}} + - {{path: /etc/postgresql-custom/pgsodium_root.key, content: {gzip_then_base64_encode(pgsodium_root_key_content)}, permissions: '0600', owner: 'postgres:postgres', encoding: gz+b64}} + - {{path: /etc/postgrest/base.conf, content: {gzip_then_base64_encode(postgrest_base_conf_content)}, permissions: '0664', encoding: gz+b64}} + - {{path: /etc/gotrue.env, content: {gzip_then_base64_encode(gotrue_env_content)}, permissions: '0664', encoding: gz+b64}} + - {{path: /etc/wal-g/config.json, content: {gzip_then_base64_encode(walg_config_json_content)}, permissions: '0664', owner: 'wal-g:wal-g', encoding: gz+b64}} + - {{path: /tmp/init.json, content: {gzip_then_base64_encode(init_json_content)}, permissions: '0600', encoding: gz+b64}} +runcmd: + - 'sudo echo \"pgbouncer\" \"postgres\" >> /etc/pgbouncer/userlist.txt' + - 'cd /tmp && aws s3 cp --region ap-southeast-1 s3://init-scripts-staging/project/init.sh .' + - 'bash init.sh "staging"' + - 'rm -rf /tmp/*' +""", + TagSpecifications=[ + { + "ResourceType": "instance", + "Tags": [{"Key": "Name", "Value": "ci-ami-test"}], + } + ], + ) + )[0] + instance.wait_until_running() + + logger = EC2InstanceConnectLogger(debug=False) + temp_key = EC2InstanceConnectKey(logger.get_logger()) + ec2ic = boto3.client("ec2-instance-connect", region_name="ap-southeast-1") + response = ec2ic.send_ssh_public_key( + InstanceId=instance.id, + InstanceOSUser="ubuntu", + SSHPublicKey=temp_key.get_pub_key(), + ) + assert response["Success"] + + # instance doesn't have public ip yet + instance.reload() + + host = testinfra.get_host( + # paramiko is an ssh backend + f"paramiko://ubuntu@{instance.public_ip_address}", + ssh_identity_file=temp_key.get_priv_key_file(), + ) + + def is_healthy(host) -> bool: + cmd = host.run("pg_isready -U postgres") + if cmd.failed is True: + return False + + cmd = host.run(f"curl http://localhost:8085/health -H 'apikey: {supabase_admin_key}'") + if cmd.failed is True: + return False + + cmd = host.run("curl http://localhost:3001/ready") + if cmd.failed is True: + return False + + cmd = host.run("curl http://localhost:8081/health") + if cmd.failed is True: + return False + + cmd = host.run("sudo kong health") + if cmd.failed is True: + return False + + cmd = host.run("printf \\\\0 > '/dev/tcp/localhost/6543'") + if cmd.failed is True: + return False + + cmd = host.run("sudo fail2ban-client status") + if cmd.failed is True: + return False + + return True + + while True: + if is_healthy(host): + break + print("waiting until healthy") + sleep(1) + + # return a testinfra connection to the instance + yield host + + # at the end of the test suite, destroy the instance + instance.terminate() + + +def test_postgrest_is_running(host): + postgrest = host.service("postgrest") + assert postgrest.is_running + + +def test_postgrest_responds_to_requests(host): + res = requests.get( + f"http://{host.backend.get_hostname()}/rest/v1/", + headers={ + "apikey": anon_key, + "authorization": f"Bearer {anon_key}", + }, + ) + assert res.ok + + +def test_postgrest_can_connect_to_db(host): + res = requests.get( + f"http://{host.backend.get_hostname()}/rest/v1/buckets", + headers={ + "apikey": service_role_key, + "authorization": f"Bearer {service_role_key}", + "accept-profile": "storage", + }, + ) + assert res.ok