YDA-5949: Add Docker setup for development

(Work in progress)
UtrechtUniversity · Oct 8, 2024 · 8aca5db · 8aca5db
1 parent 9fe153b
commit 8aca5db
Show file tree

Hide file tree

Showing 8 changed files with 550 additions and 0 deletions.
diff --git a/.github/workflows/ansible-lint.yml b/.github/workflows/ansible-lint.yml
@@ -13,6 +13,8 @@ jobs:
 
     - name: Lint Ansible Playbook
       uses: ansible/[email protected]
+      with:
+        args: '--exclude=./docker'
 
         # [optional]
         # Arguments to be passed to the ansible-lint

diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -0,0 +1,68 @@
+# This is a locally modified version of the upstream CKAN docker compose file.
+# It contains image names for pushing locally built images to a central registry at ECR
+
+# docker-compose build && docker-compose up -d
+# If "docker-compose logs ckan" shows DB not ready, run "docker-compose restart ckan" a few times.
+version: "3"
+
+volumes:
+  ckan_config:
+  ckan_home:
+  ckan_storage:
+  ckan_coveragedata:
+  pg_data:
+  solr_data:
+  nginx_config:
+  nginx_certificates:
+
+services:
+  ckan:
+     container_name: ckan
+     image: epos-msl-ckan:latest
+     links:
+       - db
+       - solr
+       - redis
+     depends_on:
+       - db
+       - solr
+       - redis
+     ports:
+       - "0.0.0.0:${CKAN_PORT}:5000"
+     environment:
+       - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan
+       - CKAN_SOLR_URL=http://solr:8983/solr/ckan
+       - CKAN_REDIS_URL=redis://redis:6379/1
+       - CKAN_SITE_URL=${CKAN_SITE_URL}
+       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+     volumes:
+       - ckan_config:/etc/ckan
+       - ckan_home:/usr/lib/ckan
+       - ckan_storage:/var/lib/ckan
+       - ckan_coveragedata:/coverage
+
+  nginx:
+    container_name: revproxy
+    image: nginx:1.27
+    volumes:
+      - nginx_config:/etc/nginx/conf.d
+      - nginx_certificates:/etc/certificates
+
+  db:
+    container_name: db
+    image: postgres:12.20
+    environment:
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - PGDATA=/var/lib/postgresql/data/db
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+
+  solr:
+    container_name: solr
+    image: epos-msl-solr:latest
+    volumes:
+      - solr_data:/opt/solr/server/solr/ckan/data
+
+  redis:
+    container_name: redis
+    image: redis:6.2
diff --git a/docker/images/ckan/Dockerfile b/docker/images/ckan/Dockerfile
@@ -0,0 +1,102 @@
+# This is a locally modified version of the upstream CKAN Dockerfile
+
+# See CKAN docs on installation from Docker Compose on usage
+FROM ubuntu:focal-20210119
+MAINTAINER Yoda team
+
+# Set timezone
+ENV TZ=UTC
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+# Setting the locale
+ENV LC_ALL=en_US.UTF-8       
+RUN adduser ckan
+RUN apt-get update
+RUN apt-get install --no-install-recommends -y locales
+RUN sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen
+RUN dpkg-reconfigure --frontend=noninteractive locales 
+RUN update-locale LANG=${LC_ALL}
+
+# Install required system packages
+RUN apt-get -q -y update \
+    && DEBIAN_FRONTEND=noninteractive apt-get -q -y upgrade \
+    && apt-get -q -y install \
+        python3.8 \
+        python3-dev \
+        python3-pip \
+        python3-venv \
+        python3-wheel \
+        libpq-dev \
+        python3-pastescript \
+        python3-virtualenv \
+        libxml2-dev \
+        libxslt-dev \
+        libgeos-dev \
+        libssl-dev \
+        libffi-dev \
+        postgresql-client \
+        build-essential \
+        git-core \
+        vim \
+        wget \
+        curl \
+        nmap \
+        sqlite3 \
+        pwgen \
+        uuid-runtime \
+    && apt-get -q clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Define environment variables
+ENV CKAN_HOME /usr/lib/ckan
+ENV CKAN_VENV $CKAN_HOME/default
+ENV CKAN_CONFIG /etc/ckan
+ENV CKAN_STORAGE_PATH=/ckanstorage
+ENV CKAN_VERSION=2.9.11
+ENV CKAN_SCHEMING_VERSION=release-2.1.0
+ENV CKAN_MSL_CORE_VERSION=1.4.0
+ENV CKAN_MSL_UTIL_VERSION=1.0.0
+
+# Create storage path
+RUN mkdir -p $CKAN_STORAGE_PATH/webassets $CKAN_STORAGE_PATH/storage
+
+# Build-time variables specified by docker-compose.yml / .env
+ARG CKAN_SITE_URL
+
+# Setup virtual environment for CKAN
+RUN mkdir -p $CKAN_VENV $CKAN_CONFIG/default && \
+    python3 -m venv $CKAN_VENV && \
+    ln -s $CKAN_VENV/bin/pip3 /usr/local/bin/ckan-pip3 &&\
+    ln -s $CKAN_VENV/bin/ckan /usr/local/bin/ckan
+ADD ckan.ini /etc/ckan/default/ckan.ini
+ADD wsgi.py /etc/ckan/default/wsgi.py
+ADD ckan-uwsgi.ini /etc/ckan/default/ckan-uwsgi.ini
+
+# Virtual environment binaries/scripts to be used first
+ENV PATH=${CKAN_VENV}/bin:${PATH}  
+ENV EPOS_MSL_FQDN=epos-msl.local
+ENV CKAN_ADMIN_PASSWORD="testtest"
+
+# Install CKAN and plugins
+RUN ckan-pip3 install -U pip && \
+    ckan-pip3 install setuptools==44.1.0 && \
+    ckan-pip3 install --upgrade pip && \
+    ckan-pip3 install wheel && \
+    ckan-pip3 install -e "git+https://github.com/ckan/ckan@ckan-${CKAN_VERSION}#egg=ckan[requirements]" && \
+    ckan-pip3 install uwsgi && \
+    ckan-pip3 install -e "git+https://github.com/ckan/ckanext-scheming@${CKAN_SCHEMING_VERSION}#egg=ckanext-scheming" && \
+    ckan-pip3 install -e "git+https://github.com/UtrechtUniversity/msl_ckan_core@${CKAN_MSL_CORE_VERSION}#egg=ckanext-msl_ckan" && \
+    ckan-pip3 install -e "git+https://github.com/UtrechtUniversity/msl_ckan_util@${CKAN_MSL_UTIL_VERSION}#egg=ckanext-msl_ckan_util" && \
+    ln -s $CKAN_VENV/src/ckan/ckan/config/who.ini $CKAN_CONFIG/who.ini && \
+    cp -v $CKAN_VENV/src/ckan/contrib/docker/ckan-entrypoint.sh /ckan-entrypoint.sh && \
+    chmod +x /ckan-entrypoint.sh && \
+    chown -R ckan:ckan $CKAN_HOME $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH $COVERAGE_DIR
+
+ADD ./ckan-entrypoint.sh /ckan-entrypoint.sh
+ENTRYPOINT ["/ckan-entrypoint.sh"]
+RUN chmod +x /ckan-entrypoint.sh
+
+USER ckan
+EXPOSE 5000
+
+CMD ["/ckan-entrypoint.sh"]
diff --git a/docker/images/ckan/ckan-entrypoint.sh b/docker/images/ckan/ckan-entrypoint.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+
+set -u
+
+check_port() {
+    local host="$1"
+    local port="$2"
+
+    echo "Checking service availability at ${host}:${port}..."
+
+    while true; do
+        # Use nmap to scan the specified port on the given host
+        nmap -p "${port}" "${host}" | grep -q "open"
+
+        # If the port is open, exit the loop
+        if [ $? -eq 0 ]; then
+            echo "Server is available at ${host}:${port}"
+            break
+        else
+            echo "Server at ${host}:${port} is not available. Retrying in 1 seconds..."
+            sleep 1
+        fi
+    done
+}
+
+## Check DB up
+check_port db 5432
+
+## Check Solr up
+check_port solr 8983
+
+## Check Redis up
+check_port redis 6379
+
+## Initialize CKAN config, database and admin account
+CKAN_CONFIG_FILE=/etc/ckan/default/ckan.ini
+CKAN_INIT_STATUS_FILE=/etc/ckan/default/.ckan_initialized
+
+if test -f "$CKAN_INIT_STATUS_FILE"
+then echo "Configuration and database already initialized."
+else echo "Initializing configuration ..."
+     BEAKER_SESSION_SECRET=$(openssl rand -base64 32)
+     APP_INSTANCE_UUID=$(uuidgen --name "$EPOS_MSL_FQDN" --namespace "@url" --sha1)
+     CKAN_DATABASE_PASSWORD=$(pwgen -n 16 -N 1)
+     CKAN_MSL_VOCABULARIES_ENDPOINT="https://${EPOS_MSL_FQDN}/webservice/api/vocabularies"
+     perl -pi.bak -e '$beaker_session_secret=$ENV{BEAKER_SESSION_SECRET}; s/BEAKER_SESSION_SECRET/$beaker_session_secret/gee' "$CKAN_CONFIG_FILE"
+     perl -pi.bak -e '$app_instance_uuid=$ENV{APP_INSTANCE_UUID}; s/APP_INSTANCE_UUID/$app_instance_uuid/gee' "$CKAN_CONFIG_FILE"
+     perl -pi.bak -e '$ckan_database_password=$ENV{CKAN_DATABASE_PASSWORD}; s/CKAN_DATABASE_PASSWORD/$ckan_database_password/gee' "$CKAN_CONFIG_FILE"
+     perl -pi.bak -e '$epos_msl_fqdn=$ENV{EPOS_MSL_FQDN}; s/EPOS_MSL_FQDN/$epos_msl_fqdn/gee' "$CKAN_CONFIG_FILE"
+     perl -pi.bak -e '$ckan_msl_vocabularies_endpoint=$ENV{CKAN_MSL_VOCABULARIES_ENDPOINT}; s/CKAN_MSL_VOCABULARIES_ENDPOINT/$ckan_msl_vocabularies_endpoint/gee' "$CKAN_CONFIG_FILE"
+     echo "Initializing database ..."
+     /usr/lib/ckan/default/bin/ckan -c "$CKAN_CONFIG_FILE" db init
+     touch "$CKAN_INIT_STATUS_FILE"
+     echo "Configuration and database initialization finished."
+fi
+
+## Start CKAN
+echo "Starting CKAN ..."
+while true
+do /usr/lib/ckan/default/bin/uwsgi -i /etc/ckan/default/ckan-uwsgi.ini
+   sleep 1
+   echo "CKAN process terminated; trying to restart ..."
+done
diff --git a/docker/images/ckan/ckan-uwsgi.ini b/docker/images/ckan/ckan-uwsgi.ini
@@ -0,0 +1,15 @@
+[uwsgi]
+
+http            =  127.0.0.1:8080
+uid             =  www-data
+gid             =  www-data
+wsgi-file       =  /etc/ckan/default/wsgi.py
+virtualenv      =  /usr/lib/ckan/default
+module          =  wsgi:application
+master          =  true
+pidfile         =  /tmp/%n.pid
+harakiri        =  50
+max-requests    =  5000
+vacuum          =  true
+callable        =  application
+strict          =  true