[WIP] Dockerise Colouring London

Dockerfile

@@ -0,0 +1,33 @@
+FROM ubuntu:18.04
+
+RUN apt-get update -y
+RUN apt-get upgrade -y
+
+RUN apt-get install -y build-essential wget
+RUN apt-get install parallel -y
+
+ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.2.1/wait /wait
+RUN chmod +x /wait
+
+RUN mkdir /colouring-london
+COPY app /colouring-london/app
+COPY migrations /colouring-london/migrations
+COPY etl /colouring-london/etl
+
+ENV NODE_VERSION=16.13.2
+RUN apt-get install -y curl
+RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash
+ENV NVM_DIR=/root/.nvm
+RUN . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION}
+RUN . "$NVM_DIR/nvm.sh" && nvm use v${NODE_VERSION}
+RUN . "$NVM_DIR/nvm.sh" && nvm alias default v${NODE_VERSION}
+ENV PATH="/root/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}"
+
+RUN npm install -g npm@latest
+
+WORKDIR ./colouring-london/app
+RUN rm -rf node_modules
+RUN npm install
+
+EXPOSE 8080
+CMD /wait && npm start

docker-compose.yml

@@ -0,0 +1,31 @@
+version: "3"
+services:
+  colouring-london:
+    container_name: colouring-london
+    build:
+      context: .
+      dockerfile: Dockerfile
+    depends_on:
+      - db
+    environment:
+      DATABASE_URL: postgres://dockeruser:postgres@postgres-colouring-london:5432/colouringlondon
+      NODE_ENV: development
+      PORT: 8080
+      APP_COOKIE_SECRET: 123456
+    ports:
+      - "8080:8080"
+  db:
+    container_name: postgres-colouring-london
+    image: ubuntu/postgres:13-21.10_beta
+    ports:
+      - "5432:5432"
+    volumes: 
+      - ./postgresdata:/var/lib/postgresql/data
+      - ./colouring-london/migrations:/colouring-london/migrations
+      - ./colouring-london/etl:/colouring-london/etl
+      - ./colouring-london/init-user-db.sh:/docker-entrypoint-initdb.d/init-user-db.sh
+    restart: always
+    environment:
+      POSTGRES_USER: dockeruser
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: colouringlondon

etl/create_building_records_cl.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+#
+# Create corresponding 'building' record with
+#     id: <building-guid>,
+#     doc: {},
+#     geom_id: <polygon-guid>
+#
+psql -d colouringlondon -c "INSERT INTO buildings ( geometry_id, ref_toid ) SELECT geometry_id, source_id from geometries;"

etl/load_geometries_cl.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+#
+# Load geometries from GeoJSON to Postgres
+# - assume postgres connection details are set in the environment using PGUSER, PGHOST etc.
+#
+: ${1?"Usage: $0 ./path/to/mastermap/dir"}
+
+mastermap_dir=$1
+
+#
+# Create 'geometry' record with
+#     id: <polygon-guid>,
+#     source_id: <toid>,
+#     geom: <geom>
+#
+find $mastermap_dir -type f -name '*.3857.csv' \
+-printf "$mastermap_dir/%f\n" | \
+parallel \
+cat {} '|' psql -d colouringlondon -c "\"COPY geometries ( geometry_geom, source_id ) FROM stdin WITH CSV HEADER;\""
+
+#
+# Delete any duplicated geometries (by TOID)
+#
+psql -d colouringlondon -c "DELETE FROM geometries a USING (
+    SELECT MIN(ctid) as ctid, source_id
+    FROM geometries
+    GROUP BY source_id
+    HAVING COUNT(*) > 1
+) b
+WHERE a.source_id = b.source_id
+AND a.ctid <> b.ctid;"

init-user-db.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+apt-get update -y
+apt-get upgrade -y
+
+apt-get install -y postgresql-contrib libpq-dev postgis
+apt-get install -y postgresql-13-postgis-3
+apt-get install -y gdal-bin libspatialindex-dev libgeos-dev libproj-dev
+
+apt-get install -y python3 python3-pip python3-dev
+
+psql -d colouringlondon -U dockeruser -c "SELECT 1 FROM pg_user WHERE usename = 'dockeruser';" | grep -q 1 ||  psql -d colouringlondon -U dockeruser -c "CREATE ROLE dockeruser SUPERUSER LOGIN PASSWORD 'postgres';"
+psql -d colouringlondon -U dockeruser -c "SELECT 1 FROM pg_database WHERE datname = 'colouringlondon';" | grep -q 1 ||  -u postgres createdb -E UTF8 -T template0 --locale=en_US.utf8 -O dockeruser colouringlondon
+
+psql -d colouringlondon -U dockeruser -c "create extension postgis;"
+psql -d colouringlondon -U dockeruser -c "create extension pgcrypto;"
+psql -d colouringlondon -U dockeruser -c "create extension pg_trgm;"
+
+ls ./colouring-london/migrations/*.up.sql 2>/dev/null | while read -r migration; do psql -d colouringlondon < $migration; done;
+
+pip install --upgrade pip
+pip install --upgrade setuptools wheel
+pip install -r ./colouring-london/etl/requirements.txt
+
+python ./colouring-london/etl/get_test_polygons.py
+./colouring-london/etl/load_geometries_cl.sh ./
+psql -d colouringlondon -U dockeruser < ./colouring-london/app/migrations/002.index-geometries.up.sql
+./create_building_records_cl.sh
+psql -d colouringlondon -U dockeruser < ./colouring-london/app/migrations/003.index-buildings.up.sql
+ls ./colouring-london/migrations/*.up.sql 2>/dev/null | while read -r migration; do psql -d colouringlondon < $migration; done;