Merge pull request #489 from carlopi/fixup_distribution #812
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Local Lakekeeper Testing | |
| on: [push, pull_request,repository_dispatch] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| shell: bash | |
| env: | |
| BASE_BRANCH: ${{ github.base_ref || (endsWith(github.ref, '_feature') && 'feature' || 'main') }} | |
| jobs: | |
| rest: | |
| name: Test against Lakekeeper Catalog | |
| runs-on: ubuntu-latest | |
| if: false # disabled until lakekeeper setup process is more stable | |
| env: | |
| VCPKG_TARGET_TRIPLET: 'x64-linux-release' | |
| GEN: ninja | |
| VCPKG_FEATURE_FLAGS: "-binarycaching" | |
| VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake | |
| PIP_BREAK_SYSTEM_PACKAGES: 1 | |
| steps: | |
| - name: Install required ubuntu packages | |
| run: | | |
| sudo apt-get update -y -qq | |
| sudo apt-get install -y -qq software-properties-common | |
| sudo add-apt-repository ppa:git-core/ppa | |
| sudo apt-get update -y -qq | |
| sudo apt-get install -y -qq \ | |
| ninja-build \ | |
| make gcc-multilib \ | |
| g++-multilib \ | |
| libssl-dev \ | |
| wget \ | |
| openjdk-8-jdk \ | |
| zip \ | |
| maven \ | |
| unixodbc-dev \ | |
| libc6-dev-i386 \ | |
| lib32readline6-dev \ | |
| libssl-dev \ | |
| libcurl4-gnutls-dev \ | |
| libexpat1-dev \ | |
| gettext \ | |
| unzip \ | |
| build-essential \ | |
| checkinstall \ | |
| libffi-dev \ | |
| curl \ | |
| libz-dev \ | |
| openssh-client | |
| sudo apt-get install -y -qq tar pkg-config | |
| sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose | |
| sudo chmod +x /usr/local/bin/docker-compose | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| submodules: 'true' | |
| - name: Setup vcpkg | |
| uses: lukka/[email protected] | |
| with: | |
| vcpkgGitCommitId: ce613c41372b23b1f51333815feb3edd87ef8a8b | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| continue-on-error: true | |
| - name: Build extension | |
| env: | |
| GEN: ninja | |
| STATIC_LIBCPP: 1 | |
| run: | | |
| make release | |
| - name: Set up for Lakekeeper | |
| run: | | |
| # install java | |
| # TODO: need a newer java version maybe? | |
| sudo apt install -y -qq openjdk-21-jre-headless | |
| sudo apt install -y -qq openjdk-21-jdk-headless | |
| sudo apt-get install -y -qq python3-venv | |
| git clone https://github.com/lakekeeper/lakekeeper.git lakekeeper | |
| - name: Start Keycloak | |
| run: | | |
| docker run -d \ | |
| --name keycloak \ | |
| -p 30080:8080 \ | |
| -e KC_BOOTSTRAP_ADMIN_USERNAME=admin \ | |
| -e KC_BOOTSTRAP_ADMIN_PASSWORD=admin \ | |
| -v $(pwd)/lakekeeper/examples/access-control-simple/keycloak/realm.json:/opt/keycloak/data/import/realm.json \ | |
| quay.io/keycloak/keycloak:26.0.7 \ | |
| start-dev --metrics-enabled=true --health-enabled=true --import-realm --verbose --log-level=INFO --features=token-exchange | |
| - name: Wait for Keycloak | |
| run: | | |
| max_attempts=30 | |
| attempt=1 | |
| while ! curl -sf "http://localhost:30080/realms/master/.well-known/openid-configuration"; do | |
| if [ $attempt -gt $max_attempts ]; then | |
| echo "Keycloak failed to initialize after $max_attempts attempts" | |
| exit 1 | |
| fi | |
| echo "Waiting for Keycloak to initialize (attempt $attempt/$max_attempts)..." | |
| sleep 5 | |
| attempt=$((attempt + 1)) | |
| done | |
| echo "Keycloak is healthy" | |
| - name: Start PostgreSQL | |
| run: | | |
| docker run -d \ | |
| --name postgres \ | |
| -p 5433:5432 \ | |
| -e POSTGRESQL_USERNAME=postgres \ | |
| -e POSTGRESQL_PASSWORD=postgres \ | |
| -e POSTGRESQL_DATABASE=postgres \ | |
| bitnami/postgresql:16.3.0 | |
| - name: Wait for PostgreSQL | |
| run: | | |
| max_attempts=30 | |
| attempt=1 | |
| while ! docker exec postgres pg_isready -U postgres -p 5432 -d postgres; do | |
| if [ $attempt -gt $max_attempts ]; then | |
| echo "PostgreSQL failed to initialize after $max_attempts attempts" | |
| exit 1 | |
| fi | |
| echo "Waiting for PostgreSQL to initialize (attempt $attempt/$max_attempts)..." | |
| sleep 5 | |
| attempt=$((attempt + 1)) | |
| done | |
| echo "PostgreSQL is healthy" | |
| - name: Run OpenFGA DB migrations | |
| run: | | |
| docker run --rm \ | |
| --network="host" \ | |
| -e OPENFGA_DATASTORE_ENGINE=postgres \ | |
| -e OPENFGA_DATASTORE_URI="postgres://postgres:postgres@localhost:5433/postgres?sslmode=disable" \ | |
| openfga/openfga:v1.8 migrate | |
| - name: Start OpenFGA | |
| run: | | |
| docker run -d \ | |
| --name openfga \ | |
| -p 8081:8081 \ | |
| -e OPENFGA_DATASTORE_ENGINE=postgres \ | |
| -e OPENFGA_DATASTORE_URI="postgres://postgres:postgres@localhost:5433/postgres?sslmode=disable" \ | |
| -e OPENFGA_AUTHN_METHOD=oidc \ | |
| -e OPENFGA_AUTHN_OIDC_ISSUER=http://localhost:30080/realms/iceberg \ | |
| -e OPENFGA_AUTHN_OIDC_AUDIENCE=openfga \ | |
| -e OPENFGA_HTTP_TLS_ENABLED=false \ | |
| --network="host" \ | |
| openfga/openfga:v1.8 run --playground-enabled=false | |
| - name: Wait for OpenFGA | |
| run: | | |
| max_attempts=30 | |
| attempt=1 | |
| while ! curl -sf http://localhost:8080/healthz; do | |
| if [ $attempt -gt $max_attempts ]; then | |
| echo "OpenFGA failed to initialize after $max_attempts attempts" | |
| exit 1 | |
| fi | |
| echo "Waiting for OpenFGA to initialize (attempt $attempt/$max_attempts)..." | |
| sleep 5 | |
| attempt=$((attempt + 1)) | |
| done | |
| echo "OpenFGA is healthy" | |
| - name: Start Minio | |
| run: | | |
| docker run -d \ | |
| --name minio \ | |
| --network="host" \ | |
| -e MINIO_ROOT_USER=minio-root-user \ | |
| -e MINIO_ROOT_PASSWORD=minio-root-password \ | |
| -e MINIO_API_PORT_NUMBER=9000 \ | |
| -e MINIO_CONSOLE_PORT_NUMBER=9001 \ | |
| -e MINIO_SCHEME=http \ | |
| -e MINIO_DEFAULT_BUCKETS=examples \ | |
| --health-cmd="mc ls local | grep examples" \ | |
| --health-interval=2s \ | |
| --health-timeout=10s \ | |
| --health-retries=2 \ | |
| --health-start-period=15s \ | |
| -p 9000:9000 \ | |
| -p 9001:9001 \ | |
| bitnami/minio:2025.4.22 | |
| - name: Wait for Minio | |
| run: | | |
| max_attempts=30 | |
| attempt=1 | |
| while ! docker exec minio mc ls local | grep examples; do | |
| if [ $attempt -gt $max_attempts ]; then | |
| echo "Minio failed to initialize after $max_attempts attempts" | |
| exit 1 | |
| fi | |
| echo "Waiting for Minio to initialize (attempt $attempt/$max_attempts)..." | |
| sleep 5 | |
| attempt=$((attempt + 1)) | |
| done | |
| echo "Minio is healthy" | |
| - name: Run Lakekeeper DB Migrations | |
| run: | | |
| docker run --rm \ | |
| --network="host" \ | |
| -e LAKEKEEPER__PG_ENCRYPTION_KEY=This-is-NOT-Secure! \ | |
| -e LAKEKEEPER__PG_DATABASE_URL_READ=postgresql://postgres:postgres@localhost:5433/postgres?sslmode=disable \ | |
| -e LAKEKEEPER__PG_DATABASE_URL_WRITE=postgresql://postgres:postgres@localhost:5433/postgres?sslmode=disable \ | |
| -e LAKEKEEPER__AUTHZ_BACKEND=openfga \ | |
| -e LAKEKEEPER__OPENFGA__ENDPOINT=http://localhost:8081 \ | |
| -e LAKEKEEPER__OPENFGA__CLIENT_ID=openfga \ | |
| -e LAKEKEEPER__OPENFGA__CLIENT_SECRET=xqE1vUrifVDKAZdLuz6JAnDxMYLdGu5z \ | |
| -e LAKEKEEPER__OPENFGA__TOKEN_ENDPOINT=http://localhost:30080/realms/iceberg/protocol/openid-connect/token \ | |
| -e RUST_LOG=info,iceberg-catalog=info,middle=trace \ | |
| quay.io/lakekeeper/catalog:latest-main migrate | |
| - name: Start Lakekeeper | |
| run: | | |
| docker run -d \ | |
| --name lakekeeper \ | |
| --network="host" \ | |
| -e LAKEKEEPER__PG_ENCRYPTION_KEY=This-is-NOT-Secure! \ | |
| -e LAKEKEEPER__PG_DATABASE_URL_READ=postgresql://postgres:postgres@localhost:5433/postgres?sslmode=disable \ | |
| -e LAKEKEEPER__PG_DATABASE_URL_WRITE=postgresql://postgres:postgres@localhost:5433/postgres?sslmode=disable \ | |
| -e LAKEKEEPER__AUTHZ_BACKEND=openfga \ | |
| -e LAKEKEEPER__OPENFGA__ENDPOINT=http://localhost:8081 \ | |
| -e LAKEKEEPER__OPENID_PROVIDER_URI=http://localhost:30080/realms/iceberg \ | |
| -e LAKEKEEPER__OPENID_AUDIENCE=lakekeeper \ | |
| -e LAKEKEEPER__OPENFGA__CLIENT_ID=openfga \ | |
| -e LAKEKEEPER__OPENFGA__CLIENT_SECRET=xqE1vUrifVDKAZdLuz6JAnDxMYLdGu5z \ | |
| -e LAKEKEEPER__OPENFGA__TOKEN_ENDPOINT=http://localhost:30080/realms/iceberg/protocol/openid-connect/token \ | |
| -e LAKEKEEPER__METRICS_PORT=9002 \ | |
| quay.io/lakekeeper/catalog serve | |
| - name: Wait for Lakekeeper | |
| run: | | |
| max_attempts=30 | |
| attempt=1 | |
| while ! docker exec lakekeeper /home/nonroot/iceberg-catalog healthcheck; do | |
| if [ $attempt -gt $max_attempts ]; then | |
| echo "Lakekeeper failed to initialize after $max_attempts attempts" | |
| exit 1 | |
| fi | |
| echo "Waiting for Lakekeeper to initialize (attempt $attempt/$max_attempts)..." | |
| sleep 5 | |
| attempt=$((attempt + 1)) | |
| done | |
| echo "Lakekeeper is healthy" | |
| - name: Populate Lakekeeper | |
| env: | |
| JAVA_HOME: /usr/lib/jvm/java-21-openjdk-amd64 | |
| run: | | |
| sudo apt-get install -y -qq python3-venv | |
| python3 -m venv . | |
| source ./bin/activate | |
| python3 -m pip install poetry | |
| python3 -m pip install pyspark==3.5.0 | |
| python3 -m pip install duckdb | |
| python3 -m pip install pandas | |
| python3 -m pip install setuptools | |
| python3 scripts/lakekeeper/setup_lakekeeper_catalog.py | |
| python3 -m scripts.data_generators.generate_data lakekeeper | |
| - name: Test with rest catalog | |
| env: | |
| LAKEKEEPER_SERVER_AVAILABLE: 1 | |
| run: | | |
| ./build/release/test/unittest "$PWD/test/*" --list-test-names-only || true | |
| ./build/release/test/unittest "$PWD/test/*" | |