diff --git a/.github/workflows/python-upstream.yaml b/.github/workflows/python-upstream.yaml new file mode 100644 index 000000000..582ecfec4 --- /dev/null +++ b/.github/workflows/python-upstream.yaml @@ -0,0 +1,216 @@ +name: Python Check - Upstream + +on: + push: + branches: + - main + pull_request: + types: [opened, reopened, synchronize, labeled] + schedule: + - cron: "0 0 * * *" # Daily “At 00:00” UTC + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +defaults: + run: + working-directory: ./icechunk-python + +jobs: + build: + name: upstream-dev + runs-on: ubuntu-latest + if: ${{ + (contains(github.event.pull_request.labels.*.name, 'test-upstream') && github.event_name == 'pull_request') + || github.event_name == 'workflow_dispatch' + || github.event_name == 'schedule' + }} + steps: + - uses: actions/checkout@v4 + - name: Install Just + run: sudo snap install --edge --classic just + - name: Stand up MinIO + run: | + docker compose up -d minio + + - name: Wait for MinIO to be ready + run: | + for _ in {1..10}; do + if curl --silent --fail http://minio:9000/minio/health/live; then + break + fi + sleep 3 + done + docker compose exec -T minio mc alias set minio http://minio:9000 minio123 minio123 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + working-directory: icechunk-python + # target: ${{ matrix.platform.target }} + args: --release --out dist --find-interpreter + sccache: 'true' + # manylinux: ${{ matrix.platform.manylinux }} # https://github.com/PyO3/maturin-action/issues/245 + + - name: setup + shell: bash + working-directory: icechunk-python + run: | + set -e + python3 -m venv .venv + source .venv/bin/activate + uv pip install --group upstream + uv pip install icechunk --no-deps --find-links dist --force-reinstall + uv pip list + + - name: mypy + shell: bash + working-directory: icechunk-python + run: | + set -e + python3 -m venv .venv + source .venv/bin/activate + mypy python + + - name: Restore cached hypothesis directory + id: restore-hypothesis-cache + uses: actions/cache/restore@v4 + with: + path: icechunk-python/.hypothesis/ + key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }} + restore-keys: | + cache-hypothesis- + + - name: describe environment + shell: bash + working-directory: icechunk-python + run: | + set -e + python3 -m venv .venv + source .venv/bin/activate + pip list + + - name: pytest + id: status + shell: bash + working-directory: icechunk-python + run: | + set -e + python3 -m venv .venv + source .venv/bin/activate + pytest -n 4 --report-log output-pytest-log.json + + # explicitly save the cache so it gets updated, also do this even if it fails. + - name: Save cached hypothesis directory + id: save-hypothesis-cache + if: always() + uses: actions/cache/save@v4 + with: + path: icechunk-python/.hypothesis/ + key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }} + + - name: Generate and publish the report + if: | + failure() + && steps.status.outcome == 'failure' + && github.event_name == 'schedule' + && github.repository_owner == 'earth-mover' + uses: xarray-contrib/issue-from-pytest-log@v1 + with: + log-path: output-pytest-log.jsonl + + + xarray-backends: + name: xarray-tests-upstream + runs-on: ubuntu-latest + if: ${{ + (contains(github.event.pull_request.labels.*.name, 'test-upstream') && github.event_name == 'pull_request') + || github.event_name == 'workflow_dispatch' + || github.event_name == 'schedule' + }} + steps: + - uses: actions/checkout@v4 + with: + path: "icechunk" + + - name: Stand up MinIO + working-directory: icechunk + run: | + docker compose up -d minio + + - name: Wait for MinIO to be ready + working-directory: icechunk + run: | + for _ in {1..10}; do + if curl --silent --fail http://minio:9000/minio/health/live; then + break + fi + sleep 3 + done + docker compose exec -T minio mc alias set minio http://minio:9000 minio123 minio123 + + - uses: actions/checkout@v4 + with: + repository: "pydata/xarray" + path: "xarray" + fetch-depth: 0 # Fetch all history for all branches and tags. + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + working-directory: icechunk/icechunk-python + args: --release --out dist --find-interpreter + sccache: 'true' + + - name: setup + shell: bash + working-directory: icechunk/icechunk-python + run: | + set -e + python3 -m venv .venv + source .venv/bin/activate + uv pip install icechunk --group upstream --find-links dist + uv pip install pytest-mypy-plugins + uv pip list + + - name: pytest + id: status + shell: bash + working-directory: icechunk/icechunk-python + env: + ICECHUNK_XARRAY_BACKENDS_TESTS: 1 + run: | + set -e + python3 -m venv .venv + source .venv/bin/activate + # pass xarray's pyproject.toml so that pytest can find the `flaky` fixture + pytest -c=../../xarray/pyproject.toml -W ignore tests/run_xarray_backends_tests.py + + - name: Generate and publish the report + if: | + failure() + && steps.status.outcome == 'failure' + && github.event_name == 'schedule' + && github.repository_owner == 'earth-mover' + uses: xarray-contrib/issue-from-pytest-log@v1 + with: + log-path: output-pytest-log.jsonl + issue-title: "Nightly Xarray backends tests failed" diff --git a/icechunk-python/pyproject.toml b/icechunk-python/pyproject.toml index 5952fc29f..90b14c175 100644 --- a/icechunk-python/pyproject.toml +++ b/icechunk-python/pyproject.toml @@ -52,6 +52,29 @@ benchmark = [ "platformdirs", ] +[dependency-groups] +upstream = [ + "boto3", + "coverage", + "mypy", + "pytest", + "pytest-cov", + "pytest-asyncio", + "pytest-xdist", + "pytest-reportlog", + "ruff", + "hypothesis", + "pandas-stubs", + "boto3-stubs[s3]", + "termcolor", + # "object-store-python @ git+https://github.com/roeap/object-store-python.git", # doesn't work with uv + "object-store-python", + "dask @ git+https://github.com/dask/dask.git", + "distributed @ git+https://github.com/dask/distributed.git", + "xarray @ git+https://github.com/pydata/xarray.git", + "zarr @ git+https://github.com/zarr-developers/zarr.git", +] + [tool.maturin] features = ["pyo3/extension-module"] module-name = "icechunk._icechunk_python"