Merge pull request #719 from OSMCha/jlow/docker-improvements

Improve Docker development workflow

.github/workflows/django.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [3.9, '3.10']
+        python-version: ['3.9', '3.10']
 
     steps:
     - uses: actions/checkout@v2
@@ -33,7 +33,7 @@ jobs:
       run: |
         coverage run manage.py test --settings=config.settings.tests
       env:
-        OAUTH_OSM_KEY: ${{ secrets.OAUTH_OSM_KEY }}
-        OAUTH_OSM_SECRET: ${{ secrets.OAUTH_OSM_SECRET }}
-        POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
-        POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
+        OAUTH2_OSM_KEY: ${{ secrets.OAUTH_OSM_KEY }}
+        OAUTH2_OSM_SECRET: ${{ secrets.OAUTH_OSM_SECRET }}
+        PGUSER: ${{ secrets.POSTGRES_USER }}
+        PGPASSWORD: ${{ secrets.POSTGRES_PASSWORD }}

.github/workflows/docker-publish.yml

@@ -2,34 +2,50 @@ name: Build Docker Image, Push to GHCR
 
 on:
   push:
-    branches:
-    - master
+    branches: [ "main" ]
+    tags: [ 'v*.*.*' ]
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: osmcha/osmcha-django
 
 jobs:
   build:
     runs-on: ubuntu-latest
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
+    # Enable BuildKit for docker builds. This enables building
+    # multi-platform images and exporting the layer cache
+    # https://github.com/docker/setup-buildx-action
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v1
+      uses: docker/setup-buildx-action@v3
 
-    - name: Login to GitHub Container Registry
-      uses: docker/login-action@v2
+    # https://github.com/docker/login-action
+    - name: Login to Registry ${{ env.REGISTRY }}
+      uses: docker/login-action@v3
       with:
-        registry: ghcr.io
+        registry: ${{ env.REGISTRY }}
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
 
+    # Extract metadata (tags, labels) for Docker
+    # https://github.com/docker/metadata-action
+    - name: Extract Docker metadata
+      id: meta
+      uses: docker/metadata-action@v5
+      with:
+        images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+    # https://github.com/docker/build-push-action
     - name: Build and push Docker image
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         context: .
         push: true
-        tags: |
-          ghcr.io/osmcha/osmcha-django:latest
-          ghcr.io/osmcha/osmcha-django:${{ github.sha }}
+        tags: ${{ steps.meta.outputs.tags }}
+        labels: ${{ steps.meta.outputs.labels }}
         cache-from: type=gha
-        cache-to: type=gha,mode=max
+        cache-to: type=gha,mode=max

Dockerfile

@@ -1,5 +1,7 @@
 FROM python:3.10-slim-bookworm
 ARG DEBIAN_FRONTEND=noninteractive
+ARG REQUIREMENTS_FILE=production.txt
+ARG DJANGO_SETTINGS_MODULE=config.settings.production
 
 RUN apt-get update -qq -y \
     && apt-get install -y curl wget python3 python3-dev python3-pip git \
@@ -10,21 +12,24 @@ RUN apt-get update -qq -y \
 
 # Requirements have to be pulled and installed here, otherwise caching won't work
 COPY ./requirements /requirements
-
-RUN pip install -r /requirements/production.txt
+RUN pip install -r /requirements/$REQUIREMENTS_FILE
 
 COPY . /app
 RUN useradd django
 RUN chown -R django:django /app
+WORKDIR /app
+
+RUN python manage.py collectstatic --noinput
 
-COPY ./compose/django/gunicorn.sh /gunicorn.sh
 COPY ./compose/django/entrypoint.sh /entrypoint.sh
-RUN sed -i 's/\r//' /entrypoint.sh \
-    && sed -i 's/\r//' /gunicorn.sh \
-    && chmod +x /entrypoint.sh \
-    && chmod +x /gunicorn.sh
+RUN chmod +x /entrypoint.sh
 
-WORKDIR /app
 USER django
+VOLUME /app/staticfiles
+
+# Default number of gunicorn worker processes. Using an env var instead of --workers
+# in the command below lets users of this docker image override the default easily.
+ENV WEB_CONCURRENCY 4
 
 ENTRYPOINT ["/entrypoint.sh"]
+CMD ["gunicorn", "config.wsgi", "-t", "120", "-b", "0.0.0.0:5000", "--access-logfile", "-"]

README.rst

@@ -50,11 +50,11 @@ DJANGO_DEFAULT_FROM_EMAIL               DEFAULT_FROM_EMAIL                n/a
 DJANGO_SERVER_EMAIL                     SERVER_EMAIL                      n/a                                       "osmcha-django <[email protected]>"
 DJANGO_EMAIL_SUBJECT_PREFIX             EMAIL_SUBJECT_PREFIX              n/a                                       "[osmcha-django] "
 DJANGO_CHANGESETS_FILTER                CHANGESETS_FILTER                 None                                      None
-POSTGRES_USER                           POSTGRES_USER                     None                                      None
-POSTGRES_PASSWORD                       POSTGRES_PASSWORD                 None                                      None
-PGHOST                                  PGHOST                            localhost                                 localhost
-OAUTH_OSM_KEY                           SOCIAL_AUTH_OPENSTREETMAP_KEY     None                                      None
-OAUTH_OSM_SECRET                        SOCIAL_AUTH_OPENSTREETMAP_SECRET  None                                      None
+PGUSER                                  DATABASES                         None                                      None
+PGPASSWORD                              DATABASES                         None                                      None
+PGHOST                                  DATABASES                         localhost                                 localhost
+OAUTH2_OSM_KEY                          SOCIAL_AUTH_OPENSTREETMAP_KEY     None                                      None
+OAUTH2_OSM_SECRET                       SOCIAL_AUTH_OPENSTREETMAP_SECRET  None                                      None
 DJANGO_ANON_USER_THROTTLE_RATE          ANON_USER_THROTTLE_RATE           None                                      30/min
 DJANGO_COMMON_USER_THROTTLE_RATE        COMMON_USER_THROTTLE_RATE         None                                      180/min
 DJANGO_NON_STAFF_USER_THROTTLE_RATE     NON_STAFF_USER_THROTTLE_RATE      3/min                                     3/min

compose/django/entrypoint.sh

@@ -1,32 +1,12 @@
-#!/bin/bash
+#!/bin/sh
 set -e
-cmd="$@"
 
-# This entrypoint is used to play nicely with the current cookiecutter configuration.
-# Since docker-compose relies heavily on environment variables itself for configuration, we'd have to define multiple
-# environment variables just to support cookiecutter out of the box. That makes no sense, so this little entrypoint
-# does all this for us.
-export REDIS_URL=redis://redis:6379
-# the official postgres image uses 'postgres' as default user if not set explictly.
-if [ -z "$POSTGRES_USER" ]; then
-    export POSTGRES_USER=postgres
-fi
-
-if [ -z "$POSTGRES_HOST" ]; then
-    export POSTGRES_HOST=postgres
-fi
-
-export PGHOST=$POSTGRES_HOST
-export POSTGRES_USER=$POSTGRES_USER
-export POSTGRES_PASSWORD=$POSTGRES_PASSWORD
-# export DATABASE_URL=postgres://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres:5432/$POSTGRES_USER
-
-function postgres_ready(){
+postgres_ready() {
 python << END
 import sys
 import psycopg2
 try:
-    conn = psycopg2.connect(dbname="$POSTGRES_USER", user="$POSTGRES_USER", password="$POSTGRES_PASSWORD", host="$POSTGRES_HOST")
+    conn = psycopg2.connect("") # use environment variables for connection
 except psycopg2.OperationalError:
     sys.exit(-1)
 sys.exit(0)
@@ -39,4 +19,4 @@ until postgres_ready; do
 done
 
 >&2 echo "Postgres is up - continuing..."
-exec $cmd
+exec $@

compose/django/start-dev.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
 python manage.py migrate
-python manage.py runserver_plus 0.0.0.0:8000
+python manage.py runserver_plus 0.0.0.0:5000

config/settings/aws_production.py

@@ -43,22 +43,6 @@
     ]),
 ]
 
-# DATABASE CONFIGURATION
-# ------------------------------------------------------------------------------
-# Raises ImproperlyConfigured exception if DATABASE_URL not in os.environ
-# DATABASES = {
-#     'default': env.db('DATABASE_URL', default='postgres:///osmcha'),
-# }
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.contrib.gis.db.backends.postgis',
-        'NAME': env('POSTGRES_DATABASE', default='osmcha'),
-        'USER': env('POSTGRES_USER'),
-        'PASSWORD': env('POSTGRES_PASSWORD'),
-        'HOST': env('PGHOST', default='localhost')
-     }
-}
-
 REST_FRAMEWORK = {
     'DEFAULT_AUTHENTICATION_CLASSES': (
         'rest_framework.authentication.TokenAuthentication',

config/settings/common.py

@@ -122,16 +122,14 @@
 # DATABASE CONFIGURATION
 # ------------------------------------------------------------------------------
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#databases
-# DATABASES = {
-#     'default': env.db('DATABASE_URL', default='postgres:///osmcha'),
-# }
 DATABASES = {
     'default': {
         'ENGINE': 'django.contrib.gis.db.backends.postgis',
-        'NAME': env('POSTGRES_DATABASE', default='osmcha'),
-        'USER': env('POSTGRES_USER'),
-        'PASSWORD': env('POSTGRES_PASSWORD'),
-        'HOST': env('PGHOST', default='localhost')
+        'HOST': env('PGHOST', default='localhost'),
+        'PORT': env('PGPORT', default='5432'),
+        'USER': env('PGUSER', default='postgres'),
+        'PASSWORD': env('PGPASSWORD', default=''),
+        'NAME': env('PGDATABASE', default='postgres'),
      }
 }
 DATABASES['default']['ATOMIC_REQUESTS'] = True
@@ -207,7 +205,7 @@
 STATIC_ROOT = str(ROOT_DIR('staticfiles'))
 
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#static-url
-STATIC_URL = '/static/'
+STATIC_URL = '/static/django/'
 
 # See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#std:setting-STATICFILES_DIRS
 STATICFILES_DIRS = []

config/settings/local.py

@@ -63,14 +63,5 @@
 # DATABASE CONFIGURATION
 # ------------------------------------------------------------------------------
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#databases
-DATABASES = {
-    'default': {
-         'ENGINE': 'django.contrib.gis.db.backends.postgis',
-         'NAME': 'osmcha',
-         'USER': env('POSTGRES_USER'),
-         'PASSWORD': env('POSTGRES_PASSWORD'),
-         'HOST': env('PGHOST', default='localhost')
-     }
-}
 
 # Your local stuff: Below this line define 3rd party library settings

config/settings/production.py

@@ -58,21 +58,6 @@
     ]),
 ]
 
-# DATABASE CONFIGURATION
-# ------------------------------------------------------------------------------
-# Raises ImproperlyConfigured exception if DATABASE_URL not in os.environ
-# DATABASES = {
-#     'default': env.db('DATABASE_URL', default='postgres:///osmcha'),
-# }
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.contrib.gis.db.backends.postgis',
-        'NAME': env('POSTGRES_DATABASE', default='osmcha'),
-        'USER': env('POSTGRES_USER'),
-        'PASSWORD': env('POSTGRES_PASSWORD'),
-        'HOST': env('PGHOST', default='localhost')
-     }
-}
 # CACHING
 # ------------------------------------------------------------------------------
 # Configured to use Redis, if you prefer another method, comment the REDIS and

config/urls.py

@@ -19,16 +19,6 @@
 
 urlpatterns = []
 
-# If static files are not intercepted by the web-server, serve them with the dev-server:
-if settings.DEBUG is False:  # if DEBUG is True it will be served automatically
-    urlpatterns += [
-        path(
-            'static/<path>',
-            static_views.serve,
-            {'document_root': settings.STATIC_ROOT}
-            ),
-        ]
-
 api_urls = [
     path(
         '{}'.format(API_BASE_URL),
@@ -90,8 +80,7 @@ def health_check(request):
         '',
         include(api_urls)
         ),
-
-    ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
+    ]
 
 if settings.DEBUG:
     # This allows the error pages to be debugged during development, just visit