Merge pull request #5 from seasketch/dev

upgrade devcontainer - Ubuntu Noble 24.04, Node v22

Dockerfile

@@ -3,15 +3,13 @@
 # github: seasketch/docker-gp-workspace
 # docker: seasketch/geoprocessing-workspace
 #----------------------------------- #
-ARG VARIANT="jammy"
+ARG VARIANT="noble"
 FROM mcr.microsoft.com/devcontainers/base:${VARIANT}
 
 # Switch to built-in non-root gp user with sudo capabilities
 USER vscode
-
 RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
 RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list
-
 # Install common dependencies
 RUN sudo apt-get update && sudo apt-get -y upgrade \
   && sudo apt-get install -y --no-install-recommends \
@@ -27,7 +25,7 @@ RUN sudo apt-get update && sudo apt-get -y upgrade \
     && sudo rm -rf /var/lib/apt/lists/*
 
 # Install nvm to user directory - based on https://github.com/nvm-sh/nvm and https://github.com/nvm-sh/nvm/blob/master/Dockerfile
-ENV NODE_VERSION v20.12.1
+ENV NODE_VERSION v22.12.0
 RUN mkdir -p /home/vscode/.nvm/
 RUN chown vscode:vscode -R "$HOME/.nvm"
 RUN echo 'export NVM_DIR="/home/vscode/.nvm"'                                       >> "$HOME/.bashrc"
@@ -39,11 +37,9 @@ RUN sudo curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.s
 # Install platform specific dependencies
 COPY installer/ /installer
 RUN sudo chmod -R 755 /installer
-
 # Setup miniconda path
 ENV PATH="~/miniconda3/bin:${PATH}"
 ARG PATH="~/miniconda3/bin:${PATH}"
-
 # Run target specific tasks
 ARG TARGETPLATFORM
-RUN /bin/bash -c "source /installer/$TARGETPLATFORM.sh"
+RUN /bin/bash -c "source /installer/$TARGETPLATFORM.sh"

Makefile

@@ -18,18 +18,9 @@ shell: build
 		/bin/bash
 
 test:
-	# TODO fix https://api.travis-ci.com/v3/job/166029093/log.txt
-	# # Test image inheritance and multistage builds
-	# # Problem: Dockerfile should be FROM ???, need to build against TAG from the build step
-	# cd tests && docker build --tag test-gdal-base-multistage --file Dockerfile.test .
-	# docker run --rm \
-	# 	--volume $(shell pwd)/:/app \
-	# 	test-gdal-base-multistage \
-	# 	/app/tests/run_multistage_tests.sh
-	# Test GDAL CLI, etc on the base image itself
 	docker run --rm \
 		--volume $(shell pwd)/:/app \
 		seasketch/geoprocessing-workspace:$(TAG) \
-		/app/tests/run_tests.sh
+		bash /app/tests/run_tests.sh
 
 

README.md

@@ -6,38 +6,39 @@ Features:
 
 - uses Ubuntu LTS image
 - includes common utilities for compiling software from source
+- includes LTS version of Node
 - includes conda (via miniconda) with a `base` environment that has Python and GDAL.  Users can adapt or create new environments
+- includes awscli for deploying geoprocessing projects to AWS and working with them.
 - can be used as a shell environment workspace for users and for heavy geo lifting by geoprocessing CLI commands
 - is built and distributed with the computing resources provided by github and dockerhub.
 
 ## Usage
 
-You should not need to work with this repository directly, unless you need to extend the `geoprocessing-workspace` image to meet your own needs.
-
-To do this, clone the repository locally, make any changes you want to the Dockerfile or related architecture-specific installer scripts, then use the `Makefile` to build and test it.  You will need to have the `make` software package already installed.
-
-Use the makefile for changing the build build by building images locally and testing them by shelling in to check it out, and by running the test suite.  Committing changes to `main` branch in Github will then rebuild and publish the container image via the `publish` Github action.
-
-- `make` builds an `amd64` Docker image and tags it as 'test', to differentiate it from any of the 'latest' images you might have installed.
-- `make test` Runs test suite on the 'test' image
-- `make shell` Starts a container using 'test' image (building it first if necessary), then opens a bash shell.  This is useful for testing that your Dockerfile changes are working as expected.
-
-## Packages and version numbers
-
-Operating system:
-* Ubuntu Jammy 22.04 LTS with Python 3.9
-
-Libraries:
-
-```
-PROJ_VERSION 8.0.1
-GDAL_VERSION 3.3.1
-SQLITE_VERSION 3.4
-```
-
-## GDAL
-
-GDAL 3.3.1 is used because it supports flatgeobuf files, and works with flatgeobuf JS library v3.17.4 that geoprocessing lib is locked on for now
+You should not need to work with this repository directly, unless you need to upgrade the `geoprocessing-workspace` image or extend it to meet your own needs.
+
+### Upgrade
+
+To upgrade the image:
+
+- install and start Docker Desktop on your local system
+- clone this repository locally and checkout the `dev` branch.
+- make changes to the Dockerfile or related architecture-specific installer scripts
+- run `make`
+  - Builds a new Docker image.  You will need to have the `make` software package already installed.
+  - this will build an image with the architecture of your system (`amd64` for `arm64`) and tags it as 'test', to differentiate it from any of the 'latest' images you might have installed.
+- run `make shell`
+  - starts a container using 'test' image (building it first if necessary), then opens a bash shell.  This is useful for testing that your Dockerfile changes are working as expected.
+  - try running `node -v` or `gdalinfo` and make sure these commands output as expected.  You can go as far as to init a new geoprocessing project and make sure you can import a new dataset and run precalc.
+- debugging
+  - if the build is failing part way through your Dockerfile or architecture-specific script, a useful way to debug is to comment out the part of the build that is not working (and everything after it).  Then run `make` to build an image up to that point.  Then run `make shell` and run commands manually to figure out what is going wrong and how to fix it.  Once you get it working, update your Dockerfile or architecture-specific scripts with the updated commands, then uncomment and `make` to try and run a full build.  Keep doing this until you get to something that works.
+- if your new image is looking ready
+  - commit and push your changes to the dev branch.
+  - The `publish unstable` Github action in `.github/workflows/publish-unstable.yml` will automatically trigger and publish a new container image with the `unstable` tag to [Docker Hub](https://hub.docker.com/r/seasketch/geoprocessing-workspace/tags).
+  - Follow instructions for [unstable testing](https://seasketch.github.io/geoprocessing/docs/next/tutorials/upgrade/#upgrade-devcontainer) in the geoprocessing-devcontainer.
+- if the unstable image is ready to go stable
+  - Open a PR from the `dev` branch to `main` and merge it.  Do not squash commits.
+  - One merge the `publish` Githunb action in `.github/workflows/publish.yml` will automatically trigger and publish a new container image with the `latest` tag.
+  - Follow instructions for [testing](https://seasketch.github.io/geoprocessing/docs/next/tutorials/upgrade/#upgrade-devcontainer) in the geoprocessing-devcontainer.
 
 ## Alternative Uses
 

installer/linux/amd64.sh

@@ -3,14 +3,16 @@
 # Put installer in current user home directory, install to home directory, add to user bash config, and remove installer
 mkdir -p ~/miniconda \
     && cd ~/miniconda \
-    && wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
+    && wget https://repo.anaconda.com/miniconda/Miniconda3-py312_24.7.1-0-Linux-x86_64.sh \
     && mkdir ~/.conda \
-    && chmod 755 Miniconda3-latest-Linux-x86_64.sh \
-    && bash Miniconda3-latest-Linux-x86_64.sh -b -p /home/vscode/miniconda3 \
+    && chmod 755 Miniconda3-py312_24.7.1-0-Linux-x86_64.sh \
+    && bash Miniconda3-py312_24.7.1-0-Linux-x86_64.sh -b -p /home/vscode/miniconda3 \
     && echo "Running $(conda --version)" && \
+    conda config --add channels defaults && \
     conda init bash && \
     . /home/vscode/.bashrc && \
     conda update conda && \
+    conda install --solver=classic conda-forge::conda-libmamba-solver conda-forge::libmamba conda-forge::libmambapy conda-forge::libarchive && \
     conda install python pip && \
     conda install -c conda-forge gdal && \
     cd ~ && \
@@ -22,3 +24,6 @@ mkdir -p ~/miniconda \
     && unzip awscliv2.zip \
     && sudo ./aws/install \
     && rm -rf ~/awscli
+
+# miniconda pinned to py312_24.7.1-0-Linux-x86_64 due to sqlite shared library not found error
+# conda-libmamba-solver workaround due to error - https://stackoverflow.com/a/78293971

installer/linux/arm64.sh

@@ -3,14 +3,16 @@
 # Put installer in current user home directory, install to home directory, add to user bash config, and remove installer
 mkdir -p ~/miniconda \
     && cd ~/miniconda \
-    && wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh \
+    && wget https://repo.anaconda.com/miniconda/Miniconda3-py312_24.7.1-0-Linux-aarch64.sh \
     && mkdir ~/.conda \
-    && chmod 755 Miniconda3-latest-Linux-aarch64.sh \
-    && bash Miniconda3-latest-Linux-aarch64.sh -b -p /home/vscode/miniconda3 \
+    && chmod 755 Miniconda3-py312_24.7.1-0-Linux-aarch64.sh \
+    && bash Miniconda3-py312_24.7.1-0-Linux-aarch64.sh -b -p /home/vscode/miniconda3 \
     && echo "Running $(conda --version)" && \
+    conda config --add channels defaults && \
     conda init bash && \
     . /home/vscode/.bashrc && \
     conda update conda && \
+    conda install --solver=classic conda-forge::conda-libmamba-solver conda-forge::libmamba conda-forge::libmambapy conda-forge::libarchive && \
     conda install pip && \
     conda install -c conda-forge gdal && \
     cd ~ && \

tests/run_tests.sh

@@ -1,12 +1,10 @@
 #!/bin/bash
 
-echo
 # echo "proj" $(proj 2>&1 | head -n 1)
 # echo "geos" $(geos-config --version)
 gdalinfo --version
-ogrinfo --formats | grep FlatGeobuf
-python --version
-# echo "node" $(node -v)
-# echo "npm" $(npm -v)
+python3 --version
+node -v
+npm -v
 
 exit 0