Skip to content

Implement loading dataset from GCP #920

Implement loading dataset from GCP

Implement loading dataset from GCP #920

Workflow file for this run

# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
name: Python application
on:
push:
branches: [ "main" ]
pull_request:
branches: [ "main" ]
permissions:
contents: read
jobs:
usl_lib:
name: usl_lib CI
runs-on: ubuntu-22.04
env:
FIRESTORE_EMULATOR_HOST: "localhost:8085"
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.11
uses: actions/setup-python@v3
with:
python-version: "3.11"
- name: Start FireStore Emulator
run: |
sudo apt-get update -qq
sudo apt-get install -y apt-transport-https ca-certificates gnupg curl
curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg
echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
sudo apt-get update && sudo apt-get install -y google-cloud-cli google-cloud-cli-firestore-emulator
gcloud emulators firestore start --host-port=localhost:8085 &
- name: Install dependencies
run: |
sudo apt-get update -qq
sudo add-apt-repository ppa:ubuntugis/ppa -y
sudo apt-get update -qq
sudo apt-get install -y gdal-bin libgdal-dev
python -m pip install --upgrade pip
cd usl_pipeline
pip install -e usl_lib[dev]
- name: Lint with flake8
run: |
flake8 usl_pipeline/usl_lib --show-source --statistics
- name: Ensure black auto-formatter has run
run: |
black usl_pipeline/usl_lib --check
- name: Test with pytest
run: |
pytest usl_pipeline/usl_lib
- name: MyPy Type Checking
run: |
mypy usl_pipeline/usl_lib
study_area_uploader:
name: Study Area Uploader CI
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.11
uses: actions/setup-python@v3
with:
python-version: "3.11"
- name: Install dependencies
run: |
sudo apt-get update -qq
sudo add-apt-repository ppa:ubuntugis/ppa -y
sudo apt-get update -qq
sudo apt-get install -y gdal-bin libgdal-dev
python -m pip install --upgrade pip
cd usl_pipeline/study_area_uploader
pip install -r requirements.txt
pip install -e .[dev]
- name: Lint with flake8
run: |
flake8 usl_pipeline/study_area_uploader --show-source --statistics
- name: Ensure black auto-formatter has run
run: |
black usl_pipeline/study_area_uploader --check
- name: Test with pytest
run: |
pytest usl_pipeline/study_area_uploader
- name: MyPy Type Checking
run: |
mypy usl_pipeline/study_area_uploader
cloud_functions:
name: Cloud Funcations CI
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.11
uses: actions/setup-python@v3
with:
python-version: "3.11"
- name: Install dependencies
# Copy the usl_lib files to sit alongside main.py to mirror the .zip we
# will use for the cloud function.
run: |
python -m pip install --upgrade pip
# Copy wheels folder to current directory for requirements install
cp -r usl_pipeline/cloud_functions/wheels .
pip install -r usl_pipeline/cloud_functions/requirements.txt
pip install black~=24.0 flake8~=7.0 flake8-docstrings~=1.7 pytest~=8.0 mypy~=1.9
- name: Lint with flake8
run: |
flake8 usl_pipeline/cloud_functions --show-source --statistics
- name: Ensure black auto-formatter has run
run: |
black usl_pipeline/cloud_functions --check
- name: Test with pytest
run: |
pytest usl_pipeline/cloud_functions
- name: MyPy Type Checking
run: |
mypy usl_pipeline/cloud_functions
usl_models:
name: USL Models CI
runs-on: ubuntu-22.04
env:
FIRESTORE_EMULATOR_HOST: "localhost:8085"
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.11
uses: actions/setup-python@v3
with:
python-version: "3.11"
- name: Start FireStore Emulator
run: |
sudo apt-get update -qq
sudo apt-get install -y apt-transport-https ca-certificates gnupg curl
curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg
echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
sudo apt-get update && sudo apt-get install -y google-cloud-cli google-cloud-cli-firestore-emulator
gcloud emulators firestore start --host-port=localhost:8085 &
- name: Install dependencies
run: |
python -m pip install --upgrade pip
cd usl_models
pip install -r requirements.txt
pip install -e .[dev]
- name: Lint with flake8
run: |
flake8 usl_models --show-source --statistics
- name: Ensure black auto-formatter has run
run: |
black usl_models --check
- name: Test with pytest
run: |
pytest usl_models
- name: Type check with MyPy
run: |
mypy usl_models