diff --git a/.github/workflows/test_changed_notebooks.yaml b/.github/workflows/test_changed_notebooks.yaml new file mode 100644 index 00000000..351f3a86 --- /dev/null +++ b/.github/workflows/test_changed_notebooks.yaml @@ -0,0 +1,106 @@ +name: Test Modified Files + +on: + pull_request: + branches: [ develop ] + +permissions: + id-token: write # Required for OIDC authentication to AWS + contents: read + +jobs: + test_files: + runs-on: ubuntu-latest + + # Automatically activate our mamba environment when it's ready + defaults: + run: + shell: bash -l {0} + + steps: + - uses: actions/checkout@v4 + with: + path: dea-notebooks + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::538673716275:role/github-actions-role-readonly + aws-region: ap-southeast-2 + role-duration-seconds: 7200 # 2 hours + + - name: Copy tide modelling files with the AWS CLI + run: | + aws s3 sync s3://dea-non-public-data/tide_models/tide_models/fes2014 tide_models/fes2014 + aws s3 sync s3://dea-non-public-data/tide_models/tide_models/hamtide tide_models/hamtide + + - name: Login to Amazon ECR Private + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Pull dea-sandbox image from ECR + run: | + docker pull 538673716275.dkr.ecr.ap-southeast-2.amazonaws.com/geoscienceaustralia/sandbox:stable + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::060378307146:role/github-actions-dea-notebooks--sandbox + aws-region: ap-southeast-2 + role-duration-seconds: 7200 # 2 hours + + - name: Get Database Credentials + run: | + username_password=$(aws ssm get-parameter --name /dea-sandbox-eks/sandbox_reader/db.creds --with-decryption --query Parameter.Value --output text) + echo DATACUBE_DB_URL=postgresql://${username_password}@localhost:5432/odc >> $GITHUB_ENV + + - name: Open Port Forward to RDS + run: | + npx basti connect \ + --custom-target-vpc vpc-086904199e505c1f6 \ + --custom-target-host db-aurora-dea-sandbox-eks-1.cos5zfpkso9m.ap-southeast-2.rds.amazonaws.com \ + --custom-target-port 5432 \ + --local-port 5432 & + npx wait-on --timeout 120000 --interval 1000 tcp:127.0.0.1:5432 + echo "PGPORT=5432" >> $GITHUB_ENV + echo "PGHOST=localhost" >> $GITHUB_ENV + + - name: Get changed notebook files + id: changed-notebooks + uses: tj-actions/changed-files@v44 + with: + files: '**/*.ipynb' + separator: ' ' + + - name: Print changed notebook files + if: steps.changed-notebooks.outputs.any_changed == 'true' + run: | + echo "Changed notebook files:" + echo "${{ steps.changed-notebooks.outputs.all_changed_files }}" + + - name: Run the Notebook tests + if: steps.changed-notebooks.outputs.any_changed == 'true' + run: | + sudo chown -R 1000:100 ./dea-notebooks + cd ./dea-notebooks + docker run --rm \ + --net=host \ + --env DATACUBE_DB_URL \ + --env AWS_SESSION_TOKEN \ + --env AWS_REGION \ + --env AWS_ACCESS_KEY_ID \ + --env AWS_SECRET_ACCESS_KEY \ + --env AWS_SESSION_TOKEN \ + --volume ${GITHUB_WORKSPACE}/dea-notebooks:/home/jovyan/dea-notebooks \ + --volume ${GITHUB_WORKSPACE}/tide_models:/var/share/tide_models \ + --env GDAL_HTTP_MAX_RETRY=3 \ + --entrypoint /bin/bash \ + 538673716275.dkr.ecr.ap-southeast-2.amazonaws.com/geoscienceaustralia/sandbox:stable \ + -c " + set -ex + set -o pipefail + cd /home/jovyan/dea-notebooks + pip3 install ./Tools + # pytest Tests/dea_tools + pytest --durations=10 --nbval-lax ${{ steps.changed-notebooks.outputs.all_changed_files }} + " \ No newline at end of file diff --git a/.github/workflows/test_notebooks.yml b/.github/workflows/test_notebooks.yml deleted file mode 100644 index c589f5a0..00000000 --- a/.github/workflows/test_notebooks.yml +++ /dev/null @@ -1,53 +0,0 @@ - -name: Test notebooks - -on: - push: - branches: [ develop, stable, nbtests ] - paths-ignore: - - '**/*.md' # ignore markdown files - - '**/*.rst' # ignore restructured text files - - '.github/**' # ignore anything in .github folder - - '!.github/workflows/test_notebooks.yml' # except test_notebooks.yml - pull_request: - branches: [ develop, stable ] - paths-ignore: - - '**/*.md' - - '**/*.rst' - - '.github/**' - - '!.github/workflows/test_notebooks.yml' - -permissions: - id-token: write # This is required for requesting the JSON web token - contents: read # This is required for actions/checkout - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - path: dea-notebooks - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: arn:aws:iam::538673716275:role/github-actions-role-readonly - aws-region: ap-southeast-2 - - - name: Copy tide modelling files with the AWS CLI - run: | - aws s3 sync s3://dea-non-public-data/tide_models/tide_models/fes2014 tide_models/fes2014 - aws s3 sync s3://dea-non-public-data/tide_models/tide_models/hamtide tide_models/hamtide - - - name: Login to Amazon ECR Private - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - - name: Set up Datacube and test - run: | - sudo chown -R 1000:100 ./dea-notebooks - cd ./dea-notebooks - CURRENT_UID=1000:100 docker compose up -d - docker compose exec -T sandbox ./dea-notebooks/Tests/setup_test_datacube.sh - docker compose exec -T sandbox ./dea-notebooks/Tests/test_notebooks.sh diff --git a/Beginners_guide/07_Intro_to_numpy.ipynb b/Beginners_guide/07_Intro_to_numpy.ipynb index 2abcc239..02ab2fd8 100644 --- a/Beginners_guide/07_Intro_to_numpy.ipynb +++ b/Beginners_guide/07_Intro_to_numpy.ipynb @@ -165,6 +165,15 @@ "a" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "1" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -1029,7 +1038,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -1043,7 +1052,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.10" + "version": "3.10.13" }, "widgets": { "application/vnd.jupyter.widget-state+json": { diff --git a/How_to_guides/Calculating_band_indices.ipynb b/How_to_guides/Calculating_band_indices.ipynb index 6e9cb78a..95161cb5 100644 --- a/How_to_guides/Calculating_band_indices.ipynb +++ b/How_to_guides/Calculating_band_indices.ipynb @@ -18,7 +18,9 @@ "source": [ "## Background\n", "Remote sensing indices are combinations of spectral bands used to highlight features in the data and the underlying landscape.\n", - "Using Digital Earth Australia's archive of analysis-ready satellite data, we can easily calculate a wide range of remote sensing indices that can be used to assist in mapping and monitoring features like vegetation and water consistently through time, or as inputs to machine learning or classification algorithms.\n" + "Using Digital Earth Australia's archive of analysis-ready satellite data, we can easily calculate a wide range of remote sensing indices that can be used to assist in mapping and monitoring features like vegetation and water consistently through time, or as inputs to machine learning or classification algorithms.\n", + "\n", + "Test change\n" ] }, { diff --git a/Tools/dea_tools/coastal.py b/Tools/dea_tools/coastal.py index c9f30d6c..2956fc60 100644 --- a/Tools/dea_tools/coastal.py +++ b/Tools/dea_tools/coastal.py @@ -19,7 +19,6 @@ Last modified: July 2024 """ - # Import required packages import os import pyproj