YDA-6000 Adapt Docker setup for EPOS-MSL cat v2

First adaptations Docker setup for version 2 of the EPOS-MSL playbook. Most important changes: - CKAN is upgraded from 2.9.x to 2.11.x - The MSL-API application will now handle front-end traffic rather than CKAN. CKAN will only be used in the back end by the MSL-API application. - The Ubuntu base images for CKAN and MSL-API have been upgraded from Ubuntu 20.04 LTS to Ubuntu 22.04 LTS.
UtrechtUniversity · Nov 28, 2024 · 2f5c29a · 2f5c29a
1 parent 45ad927
commit 2f5c29a
Show file tree

Hide file tree

Showing 9 changed files with 80 additions and 53 deletions.
diff --git a/docker/images/ckan/Dockerfile b/docker/images/ckan/Dockerfile
@@ -1,7 +1,7 @@
 # This is a locally modified version of the upstream CKAN Dockerfile
 
 # See CKAN docs on installation from Docker Compose on usage
-FROM ubuntu:focal-20210119
+FROM ubuntu:jammy
 MAINTAINER Yoda team
 
 # Set timezone
@@ -21,7 +21,7 @@ RUN update-locale LANG=${LC_ALL}
 RUN apt-get -q -y update \
     && DEBIAN_FRONTEND=noninteractive apt-get -q -y upgrade \
     && apt-get -q -y install \
-        python3.8 \
+        python3.10 \
         python3-dev \
         python3-pip \
         python3-venv \
@@ -34,6 +34,8 @@ RUN apt-get -q -y update \
         libgeos-dev \
         libssl-dev \
         libffi-dev \
+        libmagic1 \
+        libmagic-dev \
         postgresql-client \
         build-essential \
         git-core \
@@ -52,10 +54,11 @@ ENV CKAN_HOME /usr/lib/ckan
 ENV CKAN_VENV $CKAN_HOME/default
 ENV CKAN_CONFIG /etc/ckan
 ENV CKAN_STORAGE_PATH=/ckanstorage
-ENV CKAN_VERSION=2.9.11
-ENV CKAN_SCHEMING_VERSION=release-2.1.0
-ENV CKAN_MSL_CORE_VERSION=1.4.0
-ENV CKAN_MSL_UTIL_VERSION=1.0.0
+ENV CKAN_VERSION=2.11.0
+ENV CKAN_SCHEMING_VERSION=master
+ENV CKAN_SPATIAL_VERSION=master
+ENV CKAN_MSL_CORE_VERSION=development
+ENV CKAN_MSL_UTIL_VERSION=development
 
 # Create storage path
 RUN mkdir -p $CKAN_STORAGE_PATH/webassets $CKAN_STORAGE_PATH/storage
@@ -86,11 +89,11 @@ RUN ckan-pip3 install -U pip && \
     ckan-pip3 install -e "git+https://github.com/ckan/ckan@ckan-${CKAN_VERSION}#egg=ckan[requirements]" && \
     ckan-pip3 install uwsgi && \
     ckan-pip3 install -e "git+https://github.com/ckan/ckanext-scheming@${CKAN_SCHEMING_VERSION}#egg=ckanext-scheming" && \
+    ckan-pip3 install -e "git+https://github.com/ckan/ckanext-spatial@${CKAN_SPATIAL_VERSION}#egg=ckanext-spatial" && \
+    ckan-pip3 install -r /usr/lib/ckan/default/src/ckanext-spatial/requirements.txt && \
     ckan-pip3 install -e "git+https://github.com/UtrechtUniversity/msl_ckan_core@${CKAN_MSL_CORE_VERSION}#egg=ckanext-msl_ckan" && \
     ckan-pip3 install -e "git+https://github.com/UtrechtUniversity/msl_ckan_util@${CKAN_MSL_UTIL_VERSION}#egg=ckanext-msl_ckan_util" && \
     ln -s $CKAN_VENV/src/ckan/ckan/config/who.ini $CKAN_CONFIG/who.ini && \
-    cp -v $CKAN_VENV/src/ckan/contrib/docker/ckan-entrypoint.sh /ckan-entrypoint.sh && \
-    chmod +x /ckan-entrypoint.sh && \
     chown -R ckan:ckan $CKAN_HOME $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH $COVERAGE_DIR && \
     rm /usr/lib/ckan/default/src/ckan/ckan/config/solr/schema.xml && \
     ln -sf /usr/lib/ckan/default/src/ckanext-msl-ckan/ckanext/msl_ckan/config/solr/schema.xml /usr/lib/ckan/default/src/ckan/ckan/config/solr/schema.xml && \

diff --git a/docker/images/ckan/ckan-entrypoint.sh b/docker/images/ckan/ckan-entrypoint.sh
@@ -40,10 +40,12 @@ if test -f "$CKAN_INIT_STATUS_FILE"
 then echo "Configuration and database already initialized."
 else echo "Initializing configuration ..."
      export BEAKER_SESSION_SECRET=$(openssl rand -base64 32)
+     export SECRET_TOKEN_VALUE=$(openssl rand -base64 32)
      export APP_INSTANCE_UUID=$(uuidgen --name "$EPOS_MSL_FQDN" --namespace "@url" --sha1)
      export CKAN_DATABASE_PASSWORD=$(pwgen -n 16 -N 1)
      export CKAN_MSL_VOCABULARIES_ENDPOINT="https://${EPOS_MSL_FQDN}/webservice/api/vocabularies"
      perl -pi.bak -e '$beaker_session_secret=$ENV{BEAKER_SESSION_SECRET}; s/BEAKER_SESSION_SECRET/$beaker_session_secret/ge' "$CKAN_CONFIG_FILE"
+     perl -pi.bak -e '$secret_token=$ENV{SECRET_TOKEN_VALUE}; s/SECRET_TOKEN_VALUE/$secret_token/ge' "$CKAN_CONFIG_FILE"
      perl -pi.bak -e '$app_instance_uuid=$ENV{APP_INSTANCE_UUID}; s/APP_INSTANCE_UUID/$app_instance_uuid/ge' "$CKAN_CONFIG_FILE"
      perl -pi.bak -e '$ckan_database_password=$ENV{CKAN_DATABASE_PASSWORD}; s/CKAN_DATABASE_PASSWORD/$ckan_database_password/ge' "$CKAN_CONFIG_FILE"
      perl -pi.bak -e '$epos_msl_fqdn=$ENV{EPOS_MSL_FQDN}; s/EPOS_MSL_FQDN/$epos_msl_fqdn/ge' "$CKAN_CONFIG_FILE"

diff --git a/docker/images/ckan/ckan.ini b/docker/images/ckan/ckan.ini
@@ -33,6 +33,8 @@ beaker.session.key = ckan
 # time it generates a config file.
 beaker.session.secret = BEAKER_SESSION_SECRET
 
+SECRET_KEY = SECRET_TOKEN_VALUE
+
 # `ckan generate config` generates a unique value for this each time it generates
 # a config file.
 app_instance_uuid = APP_INSTANCE_UUID
@@ -122,7 +124,7 @@ ckan.redis.url = redis://redis:6379/0
 #		Add ``resource_proxy`` to enable resorce proxying and get around the
 #		same origin policy
 #ckan.plugins = stats text_view image_view recline_view
-ckan.plugins = stats text_view image_view recline_view msl_ckan scheming_datasets scheming_groups scheming_organizations msl_custom_facets msl_repeating_fields
+ckan.plugins = stats text_view image_view msl_ckan scheming_datasets scheming_groups scheming_organizations msl_custom_facets msl_repeating_fields spatial_metadata spatial_query
 
 scheming.dataset_schemas = ckanext.msl_ckan:schemas/datasets/data_publication.yml ckanext.msl_ckan:schemas/datasets/labs.json
 scheming.organization_schemas = ckanext.msl_ckan:schemas/organizations/organization.json
@@ -154,7 +156,7 @@ ckan.favicon = /base/images/ckan.ico
 ckan.gravatar_default = identicon
 ckan.preview.direct = png jpg gif
 ckan.preview.loadable = html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json
-ckan.display_timezone = server
+ckan.display_timezone = Europe/Amsterdam
 
 # package_hide_extras = for_search_index_only
 #package_edit_return_url = http://another.frontend/dataset/<NAME>

diff --git a/docker/images/msl-api/Dockerfile b/docker/images/msl-api/Dockerfile
@@ -1,7 +1,7 @@
 # This is an image for running the MSL-API application, that is part of the EPOS-MSL catalog
 
 # See CKAN docs on installation from Docker Compose on usage
-FROM ubuntu:focal-20210119
+FROM ubuntu:jammy
 MAINTAINER Yoda team
 
 # Set timezone
@@ -31,16 +31,16 @@ RUN apt-get -q -y update \
        software-properties-common \
     && apt-add-repository ppa:ondrej/php \
     && apt-get -q -y install \
-       php8.0 \
-       php8.0-bcmath \
-       php8.0-common \
-       php8.0-curl \
-       php8.0-fpm \
-       php8.0-gd \
-       php8.0-mbstring \
-       php8.0-mysql \
-       php8.0-xml \
-       php8.0-zip \
+       php8.3 \
+       php8.3-bcmath \
+       php8.3-common \
+       php8.3-curl \
+       php8.3-fpm \
+       php8.3-gd \
+       php8.3-mbstring \
+       php8.3-mysql \
+       php8.3-xml \
+       php8.3-zip \
        mysql-client \
        supervisor \
     && apt-get -q clean \
@@ -55,15 +55,17 @@ ADD msl-api-entrypoint.sh /msl-api-entrypoint.sh
 RUN chmod +x /msl-api-entrypoint.sh
 
 RUN cd /var/www && \
-    git clone https://github.com/utrechtUniversity/msl_api && \
+    git clone https://github.com/utrechtUniversity/msl_api --branch development && \
     chown -R www-data:www-data msl_api
 USER www-data
 ADD msl-api.env /var/www/msl_api/.env
-ADD supervisord.conf /etc/supervisor/conf.d/mslapi.conf
+ADD supervisord.conf /var/www/msl-api-supervisord.conf
+ADD supervisord-webserver-only.conf /var/www/msl-api-supervisord-webserver-only.conf
 RUN cd /var/www/msl_api && \
-    /usr/bin/php8.0 /usr/local/bin/composer2 install
+    /usr/bin/php8.3 /usr/local/bin/composer2 install
 USER root
 RUN chown www-data /var/www/msl_api/.env && \
-    chmod 0664 /var/www/msl_api/.env
+    chmod 0664 /var/www/msl_api/.env && \
+    mkdir /run/php
 
 CMD ["/msl-api-entrypoint.sh"]
diff --git a/docker/images/msl-api/msl-api-entrypoint.sh b/docker/images/msl-api/msl-api-entrypoint.sh
@@ -43,11 +43,11 @@ FLUSH PRIVILEGES;
          cd /var/www/msl_api
 	 # Initialize the MSL-API application
 	 set -x
-	 sudo -u www-data /usr/bin/php8.0 artisan key:generate
-	 sudo -u www-data /usr/bin/php8.0 artisan config:cache
-         sudo -u www-data /usr/bin/php8.0 artisan migrate --force
-         sudo -u www-data /usr/bin/php8.0 artisan db:seed --force
-         sudo -u www-data /usr/bin/php8.0 artisan storage:link
+	 sudo -u www-data /usr/bin/php8.3 artisan key:generate
+	 sudo -u www-data /usr/bin/php8.3 artisan config:cache
+         sudo -u www-data /usr/bin/php8.3 artisan migrate --force
+         sudo -u www-data /usr/bin/php8.3 artisan db:seed --force
+         sudo -u www-data /usr/bin/php8.3 artisan storage:link
 	 set +x
 	 touch "$SIGNALFILE"
     elif [ "$MSLAPI_ROLE" == "QUEUE_WORKER" ]
@@ -64,14 +64,18 @@ fi
 ## Run main process
 if [ "$MSLAPI_ROLE" == "QUEUE_WORKER" ]
 then while true
-     do sudo -u www-data /usr/bin/php8.0 /var/www/msl_api/artisan queue:work  --rest=1 --tries=3 --timeout=300
+     do echo "Starting MSL-API queue worker ..."
+	sudo -u www-data /usr/bin/php8.3 /var/www/msl_api/artisan queue:work  --rest=1 --tries=3 --timeout=300
 	sleep 3
 	echo "Restarting queue worker after exit..."
      done
 elif [ "$MSLAPI_ROLE" == "WEBSERVER" ]
-then /usr/sbin/nginx -g 'daemon off;'
+then echo "Starting web server using supervisord ..."
+     cp /var/www/msl-api-supervisord-webserver-only.conf /etc/supervisor/conf.d/mslapi.conf
+     /usr/bin/supervisord
 elif [ "$MSLAPI_ROLE" == "BOTH" ]
 then echo "Starting both web server and queue worker using supervisord..."
+     cp /var/www/msl-api-supervisord.conf /etc/supervisor/conf.d/mslapi.conf
      /usr/bin/supervisord
 else echo "Error: unknown MSL API role: $MSLAPI_ROLE"
 fi
diff --git a/docker/images/msl-api/msl-api.site b/docker/images/msl-api/msl-api.site
@@ -6,17 +6,12 @@ server {
     listen [::]:80 default_server;
 
     location / {
-        # This server only serves MSL-API traffic
-        return 404;
-    }
-
-    location /webservice/ {
         alias     /var/www/msl_api/public;
         try_files /index.php =404;
         autoindex  on;
 
         fastcgi_split_path_info ^(.+\.php)(/.+)$;
-        fastcgi_pass            unix:/run/php/php8.0-fpm.sock;
+        fastcgi_pass            unix:/run/php/php8.3-fpm.sock;
         fastcgi_index           index.php;
 
         include fastcgi_params;

diff --git a/docker/images/msl-api/supervisord-webserver-only.conf b/docker/images/msl-api/supervisord-webserver-only.conf
@@ -0,0 +1,23 @@
+[supervisord]
+nodaemon=true
+logfile=/var/log/supervisord.log
+pidfile=/var/run/supervisord.pid
+
+[program:nginx]
+command=/usr/sbin/nginx -g 'daemon off;'
+autostart=true
+autorestart=true
+stdout_logfile=/var/log/nginx-supervisord.log
+stderr_logfile=/var/log/nginx-supervisord-error.log
+user=root
+
+[program:php-fpm]
+command=/usr/sbin/php-fpm8.3 --nodaemonize --fpm-config /etc/php/8.3/fpm/php-fpm.conf
+autostart=true
+autorestart=true
+stdout_logfile=/var/log/php-fpm-supervisord.log
+stderr_logfile=/var/log/php-fpm-supervisord-error.log
+user=root
+
+[group:msl-api]
+programs=nginx,php-fpm
diff --git a/docker/images/msl-api/supervisord.conf b/docker/images/msl-api/supervisord.conf
@@ -11,14 +11,22 @@ stdout_logfile=/var/log/nginx-supervisord.log
 stderr_logfile=/var/log/nginx-supervisord-error.log
 user=root
 
+[program:php-fpm]
+command=/usr/sbin/php-fpm8.3 --nodaemonize --fpm-config /etc/php/8.3/fpm/php-fpm.conf
+autostart=true
+autorestart=true
+stdout_logfile=/var/log/php-fpm-supervisord.log
+stderr_logfile=/var/log/php-fpm-supervisord-error.log
+user=root
+
 [program:laravel-queue]
-command=/usr/bin/php8.0 /var/www/msl_api/artisan queue:work  --rest=1 --tries=3 --timeout=300
+command=/usr/bin/php8.3 /var/www/msl_api/artisan queue:work  --rest=1 --tries=3 --timeout=300
 autostart=true
 autorestart=true
 stdout_logfile=/var/log/laravel-queue-supervisord.log
 stderr_logfile=/var/log/laravel-queue-supervisord-error.log
 directory=/var/www/msl_api
 user=www-data
 
-[group:laravel]
-programs=nginx,laravel-queue
+[group:msl-api]
+programs=nginx,php-fpm,laravel-queue
diff --git a/docker/images/nginx/ckan.site b/docker/images/nginx/ckan.site
@@ -42,18 +42,6 @@ server {
     client_max_body_size 100M;
 
     location / {
-        proxy_pass http://ckan:8080/;
-        proxy_set_header Host $host;
-        proxy_cache cache;
-        proxy_cache_bypass $cookie_auth_tkt;
-        proxy_no_cache $cookie_auth_tkt;
-        proxy_cache_valid 30m;
-        proxy_cache_key $host$scheme$proxy_host$request_uri;
-        # In emergency comment out line to force caching
-        # proxy_ignore_headers X-Accel-Expires Expires Cache-Control;
-    }
-
-    location /webservice/ {
         proxy_pass http://mslapi_webserver:80/;
         proxy_set_header Host $host;
         proxy_cache cache;