YDA-5949: Add Docker setup for development

(Work in progress)

.ansible-lint

@@ -1,3 +1,6 @@
+exclude_paths:
+  - ./.github
+  - ./docker
 skip_list:
   - no-changed-when  # Commands should not change things if nothing needs doing
   - no-handler  # Tasks that run when changed should likely be handlers

docker/.env

@@ -0,0 +1,47 @@
+# Variables in this file will be substituted into docker-compose.yml
+# Save a copy of this file as .env and insert your own values.
+# Verify correct substitution with "docker-compose config"
+# If variables are newly added or enabled, please delete and rebuild the images to pull in changes:
+# docker-compose down
+# docker rmi -f docker_ckan docker_db
+# docker rmi $(docker images -f dangling=true -q)
+# docker-compose build
+# docker-compose up -d
+# docker-compose restart ckan # give the db service time to initialize the db cluster on first run
+
+# Image: ckan
+CKAN_SITE_ID=default
+
+EPOS_MSL_HOST=epos-msl.ckan
+#
+# On AWS, your CKAN_SITE_URL is the output of:
+# curl -s http://169.254.169.254/latest/meta-data/public-hostname
+# CKAN_SITE_URL=http://ec2-xxx-xxx-xxx-xxx.ap-southeast-2.compute.amazonaws.com
+# On OSX with Docker for Mac, your CKAN_SITE_URL is
+# CKAN_SITE_URL=http://docker.for.mac.localhost:5000
+# When running locally, CKAN_SITE_URL must contain the port
+CKAN_SITE_URL=http://localhost:5000
+#
+# CKAN_PORT must be available on the host: sudo netstat -na
+# To apply change: docker-compose down && docker rmi docker_ckan && docker-compose build ckan
+CKAN_PORT=5000
+#
+# Email settings
+CKAN_SMTP_SERVER=smtp.corporateict.domain:25
+CKAN_SMTP_STARTTLS=True
+CKAN_SMTP_USER=user
+CKAN_SMTP_PASSWORD=pass
+CKAN_SMTP_MAIL_FROM=ckan@localhost
+#
+# Image: db
+POSTGRES_PASSWORD=ckan
+#
+# POSTGRES_PORT must be available on the host: sudo netstat -na | grep 5432
+# To apply change: docker-compose down && docker rmi docker_db docker_ckan && docker-compose build
+POSTGRES_PORT=5432
+#
+# The datastore database will be created in the db container as docs
+# Readwrite user/pass will be ckan:POSTGRES_PASSWORD
+# Readonly user/pass will be datastore_ro:DATASTORE_READONLY_PASSWORD
+DATASTORE_READONLY_PASSWORD=datastore
+

docker/README.md

@@ -0,0 +1,26 @@
+# EPOS-MSL Docker development setup
+
+## Building the images
+
+The images of the EPOS-MSL are not yet available in a registry, so you'll have to build them locally first.
+
+```
+./build-local-images.sh
+```
+
+## Using the Docker setup
+
+First add an entry to your `/etc/hosts` file (or equivalent) so that queries for the development setup
+interface resolve to your loopback interface. For example:
+
+```
+127.0.0.1 epos-msl.ckan
+```
+
+Start the Docker Compose setup:
+```
+docker compose up
+```
+
+Wait until CKAN has started. Then navigate to [https://epos-msl.ckan](https://epos-msl.ckan) in your browser. The
+development VM runs with self-signed certificates, so you'll need to accept the security warning.

docker/build-local-images.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+set -e
+set -x
+
+cd images/ckan
+./build.sh
+cd ../nginx
+./build.sh
+cd ../solr
+./build.sh
+
+set +x
+echo "Building images completed."

docker/docker-compose.yml

@@ -0,0 +1,74 @@
+# This is a locally modified version of the upstream CKAN docker compose file.
+# It contains image names for pushing locally built images to a central registry at ECR
+
+# docker-compose build && docker-compose up -d
+# If "docker-compose logs ckan" shows DB not ready, run "docker-compose restart ckan" a few times.
+version: "3"
+
+volumes:
+  ckan_config:
+  ckan_home:
+  ckan_storage:
+  ckan_coveragedata:
+  pg_data:
+  solr_data:
+  nginx_config:
+  nginx_certificates:
+
+services:
+  ckan:
+     container_name: ckan
+     image: epos-msl-ckan:latest
+     links:
+       - db
+       - solr
+       - redis
+     depends_on:
+       - db
+       - solr
+       - redis
+     ports:
+       - "0.0.0.0:${CKAN_PORT}:5000"
+     environment:
+       - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan_default
+       - CKAN_SOLR_URL=http://solr:8983/solr/ckan
+       - CKAN_REDIS_URL=redis://redis:6379/1
+       - CKAN_SITE_URL=${CKAN_SITE_URL}
+       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+     volumes:
+       - ckan_config:/etc/ckan
+       - ckan_home:/usr/lib/ckan
+       - ckan_storage:/var/lib/ckan
+       - ckan_coveragedata:/coverage
+
+  nginx:
+    container_name: nginx
+    image: epos-msl-nginx:latest
+    environment:
+      - EPOS_MSL_HOST=${EPOS_MSL_HOST}
+    ports:
+      - "18443:443"
+    volumes:
+      - nginx_config:/etc/nginx/conf.d
+      - nginx_certificates:/etc/certificates
+
+  db:
+    container_name: db
+    image: postgres:12.20
+    environment:
+      - POSTGRES_DB=ckan_default
+      - POSTGRES_USER=ckan
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - PGDATA=/var/lib/postgresql/data/db
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+
+  solr:
+    container_name: solr
+    image: epos-msl-solr:latest
+    volumes:
+      - solr_data:/opt/solr/server/solr/ckan/data
+
+  redis:
+    container_name: redis
+    image: redis:6.2

docker/images/ckan/Dockerfile

@@ -0,0 +1,106 @@
+# This is a locally modified version of the upstream CKAN Dockerfile
+
+# See CKAN docs on installation from Docker Compose on usage
+FROM ubuntu:focal-20210119
+MAINTAINER Yoda team
+
+# Set timezone
+ENV TZ=UTC
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+# Setting the locale
+ENV LC_ALL=en_US.UTF-8
+RUN adduser ckan
+RUN apt-get update
+RUN apt-get install --no-install-recommends -y locales
+RUN sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen
+RUN dpkg-reconfigure --frontend=noninteractive locales
+RUN update-locale LANG=${LC_ALL}
+
+# Install required system packages
+RUN apt-get -q -y update \
+    && DEBIAN_FRONTEND=noninteractive apt-get -q -y upgrade \
+    && apt-get -q -y install \
+        python3.8 \
+        python3-dev \
+        python3-pip \
+        python3-venv \
+        python3-wheel \
+        libpq-dev \
+        python3-pastescript \
+        python3-virtualenv \
+        libxml2-dev \
+        libxslt-dev \
+        libgeos-dev \
+        libssl-dev \
+        libffi-dev \
+        postgresql-client \
+        build-essential \
+        git-core \
+        vim \
+        wget \
+        curl \
+        nmap \
+        sqlite3 \
+        pwgen \
+        uuid-runtime \
+    && apt-get -q clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Define environment variables
+ENV CKAN_HOME /usr/lib/ckan
+ENV CKAN_VENV $CKAN_HOME/default
+ENV CKAN_CONFIG /etc/ckan
+ENV CKAN_STORAGE_PATH=/ckanstorage
+ENV CKAN_VERSION=2.9.11
+ENV CKAN_SCHEMING_VERSION=release-2.1.0
+ENV CKAN_MSL_CORE_VERSION=1.4.0
+ENV CKAN_MSL_UTIL_VERSION=1.0.0
+
+# Create storage path
+RUN mkdir -p $CKAN_STORAGE_PATH/webassets $CKAN_STORAGE_PATH/storage
+
+# Build-time variables specified by docker-compose.yml / .env
+ARG CKAN_SITE_URL
+
+# Setup virtual environment for CKAN
+RUN mkdir -p $CKAN_VENV $CKAN_CONFIG/default && \
+    python3 -m venv $CKAN_VENV && \
+    ln -s $CKAN_VENV/bin/pip3 /usr/local/bin/ckan-pip3 &&\
+    ln -s $CKAN_VENV/bin/ckan /usr/local/bin/ckan
+ADD ckan.ini /etc/ckan/default/ckan.ini
+ADD who.ini /etc/ckan/default/who.ini
+ADD wsgi.py /etc/ckan/default/wsgi.py
+ADD ckan-uwsgi.ini /etc/ckan/default/ckan-uwsgi.ini
+
+# Virtual environment binaries/scripts to be used first
+ENV PATH=${CKAN_VENV}/bin:${PATH}
+ENV EPOS_MSL_FQDN=epos-msl.local
+ENV CKAN_ADMIN_PASSWORD="testtest"
+
+# Install CKAN and plugins
+RUN ckan-pip3 install -U pip && \
+    ckan-pip3 install setuptools==44.1.0 && \
+    ckan-pip3 install --upgrade pip && \
+    ckan-pip3 install wheel && \
+    ckan-pip3 install -e "git+https://github.com/ckan/ckan@ckan-${CKAN_VERSION}#egg=ckan[requirements]" && \
+    ckan-pip3 install uwsgi && \
+    ckan-pip3 install -e "git+https://github.com/ckan/ckanext-scheming@${CKAN_SCHEMING_VERSION}#egg=ckanext-scheming" && \
+    ckan-pip3 install -e "git+https://github.com/UtrechtUniversity/msl_ckan_core@${CKAN_MSL_CORE_VERSION}#egg=ckanext-msl_ckan" && \
+    ckan-pip3 install -e "git+https://github.com/UtrechtUniversity/msl_ckan_util@${CKAN_MSL_UTIL_VERSION}#egg=ckanext-msl_ckan_util" && \
+    ln -s $CKAN_VENV/src/ckan/ckan/config/who.ini $CKAN_CONFIG/who.ini && \
+    cp -v $CKAN_VENV/src/ckan/contrib/docker/ckan-entrypoint.sh /ckan-entrypoint.sh && \
+    chmod +x /ckan-entrypoint.sh && \
+    chown -R ckan:ckan $CKAN_HOME $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH $COVERAGE_DIR && \
+    rm /usr/lib/ckan/default/src/ckan/ckan/config/solr/schema.xml && \
+    ln -sf /usr/lib/ckan/default/src/ckanext-msl-ckan/ckanext/msl_ckan/config/solr/schema.xml /usr/lib/ckan/default/src/ckan/ckan/config/solr/schema.xml && \
+    perl -n -i.bak -e 'print unless /defaultSearchField/ or /solrQueryParser/' /usr/lib/ckan/default/src/ckan/ckan/config/solr/schema.xml
+
+ADD ./ckan-entrypoint.sh /ckan-entrypoint.sh
+ENTRYPOINT ["/ckan-entrypoint.sh"]
+RUN chmod +x /ckan-entrypoint.sh
+
+USER ckan
+EXPOSE 8080
+
+CMD ["/ckan-entrypoint.sh"]

docker/images/ckan/build.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+docker build -t epos-msl-ckan:latest .

docker/images/ckan/ckan-entrypoint.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+set -u
+
+check_port() {
+    local host="$1"
+    local port="$2"
+
+    echo "Checking service availability at ${host}:${port}..."
+
+    while true; do
+        # Use nmap to scan the specified port on the given host
+        nmap -p "${port}" "${host}" | grep -q "open"
+
+        # If the port is open, exit the loop
+        if [ $? -eq 0 ]; then
+            echo "Server is available at ${host}:${port}"
+            break
+        else
+            echo "Server at ${host}:${port} is not available. Retrying in 1 seconds..."
+            sleep 1
+        fi
+    done
+}
+
+## Check DB up
+check_port db 5432
+
+## Check Solr up
+check_port solr 8983
+
+## Check Redis up
+check_port redis 6379
+
+## Initialize CKAN config, database and admin account
+CKAN_CONFIG_FILE=/etc/ckan/default/ckan.ini
+CKAN_INIT_STATUS_FILE=/etc/ckan/default/.ckan_initialized
+
+if test -f "$CKAN_INIT_STATUS_FILE"
+then echo "Configuration and database already initialized."
+else echo "Initializing configuration ..."
+     export BEAKER_SESSION_SECRET=$(openssl rand -base64 32)
+     export APP_INSTANCE_UUID=$(uuidgen --name "$EPOS_MSL_FQDN" --namespace "@url" --sha1)
+     export CKAN_DATABASE_PASSWORD=$(pwgen -n 16 -N 1)
+     export CKAN_MSL_VOCABULARIES_ENDPOINT="https://${EPOS_MSL_FQDN}/webservice/api/vocabularies"
+     perl -pi.bak -e '$beaker_session_secret=$ENV{BEAKER_SESSION_SECRET}; s/BEAKER_SESSION_SECRET/$beaker_session_secret/ge' "$CKAN_CONFIG_FILE"
+     perl -pi.bak -e '$app_instance_uuid=$ENV{APP_INSTANCE_UUID}; s/APP_INSTANCE_UUID/$app_instance_uuid/ge' "$CKAN_CONFIG_FILE"
+     perl -pi.bak -e '$ckan_database_password=$ENV{CKAN_DATABASE_PASSWORD}; s/CKAN_DATABASE_PASSWORD/$ckan_database_password/ge' "$CKAN_CONFIG_FILE"
+     perl -pi.bak -e '$epos_msl_fqdn=$ENV{EPOS_MSL_FQDN}; s/EPOS_MSL_FQDN/$epos_msl_fqdn/ge' "$CKAN_CONFIG_FILE"
+     perl -pi.bak -e '$ckan_msl_vocabularies_endpoint=$ENV{CKAN_MSL_VOCABULARIES_ENDPOINT}; s/CKAN_MSL_VOCABULARIES_ENDPOINT/$ckan_msl_vocabularies_endpoint/ge' "$CKAN_CONFIG_FILE"
+     echo "Initializing database ..."
+     /usr/lib/ckan/default/bin/ckan -c "$CKAN_CONFIG_FILE" db init
+     /usr/lib/ckan/default/bin/ckan -c "$CKAN_CONFIG_FILE" user add ckanadmin password="$CKAN_ADMIN_PASSWORD" email=ckanadmin@localhost name=ckanadmin
+     /usr/lib/ckan/default/bin/ckan -c "$CKAN_CONFIG_FILE" sysadmin add ckanadmin
+     touch "$CKAN_INIT_STATUS_FILE"
+     echo "Configuration and database initialization finished."
+fi
+
+## Start CKAN
+echo "Starting CKAN ..."
+while true
+do /usr/lib/ckan/default/bin/uwsgi -i /etc/ckan/default/ckan-uwsgi.ini
+   sleep 1
+   echo "CKAN process terminated; trying to restart ..."
+done

docker/images/ckan/ckan-uwsgi.ini

@@ -0,0 +1,15 @@
+[uwsgi]
+
+http            =  0.0.0.0:8080
+uid             =  www-data
+gid             =  www-data
+wsgi-file       =  /etc/ckan/default/wsgi.py
+virtualenv      =  /usr/lib/ckan/default
+module          =  wsgi:application
+master          =  true
+pidfile         =  /tmp/%n.pid
+harakiri        =  50
+max-requests    =  5000
+vacuum          =  true
+callable        =  application
+strict          =  true