Implement dataset loading from Google Cloud Storage #912
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow will install Python dependencies, run tests and lint with a single version of Python | |
| # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python | |
| name: Python application | |
| on: | |
| push: | |
| branches: [ "main" ] | |
| pull_request: | |
| branches: [ "main" ] | |
| permissions: | |
| contents: read | |
| jobs: | |
| usl_lib: | |
| name: usl_lib CI | |
| runs-on: ubuntu-22.04 | |
| env: | |
| FIRESTORE_EMULATOR_HOST: "localhost:8085" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python 3.11 | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: "3.11" | |
| - name: Start FireStore Emulator | |
| run: | | |
| sudo apt-get update -qq | |
| sudo apt-get install -y apt-transport-https ca-certificates gnupg curl | |
| curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg | |
| echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list | |
| sudo apt-get update && sudo apt-get install -y google-cloud-cli google-cloud-cli-firestore-emulator | |
| gcloud emulators firestore start --host-port=localhost:8085 & | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get update -qq | |
| sudo add-apt-repository ppa:ubuntugis/ppa -y | |
| sudo apt-get update -qq | |
| sudo apt-get install -y gdal-bin libgdal-dev | |
| python -m pip install --upgrade pip | |
| cd usl_pipeline | |
| pip install -e usl_lib[dev] | |
| - name: Lint with flake8 | |
| run: | | |
| flake8 usl_pipeline/usl_lib --show-source --statistics | |
| - name: Ensure black auto-formatter has run | |
| run: | | |
| black usl_pipeline/usl_lib --check | |
| - name: Test with pytest | |
| run: | | |
| pytest usl_pipeline/usl_lib | |
| - name: MyPy Type Checking | |
| run: | | |
| mypy usl_pipeline/usl_lib | |
| study_area_uploader: | |
| name: Study Area Uploader CI | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python 3.11 | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: "3.11" | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get update -qq | |
| sudo add-apt-repository ppa:ubuntugis/ppa -y | |
| sudo apt-get update -qq | |
| sudo apt-get install -y gdal-bin libgdal-dev | |
| python -m pip install --upgrade pip | |
| cd usl_pipeline/study_area_uploader | |
| pip install -r requirements.txt | |
| pip install -e .[dev] | |
| - name: Lint with flake8 | |
| run: | | |
| flake8 usl_pipeline/study_area_uploader --show-source --statistics | |
| - name: Ensure black auto-formatter has run | |
| run: | | |
| black usl_pipeline/study_area_uploader --check | |
| - name: Test with pytest | |
| run: | | |
| pytest usl_pipeline/study_area_uploader | |
| - name: MyPy Type Checking | |
| run: | | |
| mypy usl_pipeline/study_area_uploader | |
| cloud_functions: | |
| name: Cloud Funcations CI | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python 3.11 | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: "3.11" | |
| - name: Install dependencies | |
| # Copy the usl_lib files to sit alongside main.py to mirror the .zip we | |
| # will use for the cloud function. | |
| run: | | |
| python -m pip install --upgrade pip | |
| # Copy wheels folder to current directory for requirements install | |
| cp -r usl_pipeline/cloud_functions/wheels . | |
| pip install -r usl_pipeline/cloud_functions/requirements.txt | |
| pip install black~=24.0 flake8~=7.0 flake8-docstrings~=1.7 pytest~=8.0 mypy~=1.9 | |
| - name: Lint with flake8 | |
| run: | | |
| flake8 usl_pipeline/cloud_functions --show-source --statistics | |
| - name: Ensure black auto-formatter has run | |
| run: | | |
| black usl_pipeline/cloud_functions --check | |
| - name: Test with pytest | |
| run: | | |
| pytest usl_pipeline/cloud_functions | |
| - name: MyPy Type Checking | |
| run: | | |
| mypy usl_pipeline/cloud_functions | |
| usl_models: | |
| name: USL Models CI | |
| runs-on: ubuntu-22.04 | |
| env: | |
| FIRESTORE_EMULATOR_HOST: "localhost:8085" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python 3.11 | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: "3.11" | |
| - name: Start FireStore Emulator | |
| run: | | |
| sudo apt-get update -qq | |
| sudo apt-get install -y apt-transport-https ca-certificates gnupg curl | |
| curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg | |
| echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list | |
| sudo apt-get update && sudo apt-get install -y google-cloud-cli google-cloud-cli-firestore-emulator | |
| gcloud emulators firestore start --host-port=localhost:8085 & | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| cd usl_models | |
| pip install -r requirements.txt | |
| pip install -e .[dev] | |
| - name: Lint with flake8 | |
| run: | | |
| flake8 usl_models --show-source --statistics | |
| - name: Ensure black auto-formatter has run | |
| run: | | |
| black usl_models --check | |
| - name: Test with pytest | |
| run: | | |
| pytest usl_models | |
| - name: Type check with MyPy | |
| run: | | |
| mypy usl_models |