Updating to Django 5 and Ubuntu 24.04

Containerfile

@@ -1,21 +1,26 @@
-FROM ubuntu:22.04
+FROM ubuntu:24.04
 
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1
 
-RUN apt-get update && apt-get install -y tzdata && apt install -y python3.10 python3-pip python3-dev python3-numpy libpq-dev nginx libmysqlclient-dev build-essential libsasl2-dev libldap2-dev libssl-dev xmlsec1 gettext
+RUN apt-get update && apt-get install -y tzdata && apt install -y python3.12 python3-pip python3-dev python3.12-venv libpq-dev nginx libmysqlclient-dev pkg-config build-essential libsasl2-dev libldap2-dev libssl-dev xmlsec1 gettext git
 
 WORKDIR /opt/userportal
 
 COPY requirements.txt ./
 
-RUN pip3 install --upgrade pip && \
-    pip3 install --no-cache-dir -r requirements.txt
+RUN python3 -m venv /opt/userportal-env && \
+    /opt/userportal-env/bin/pip install --upgrade pip && \
+    /opt/userportal-env/bin/pip install --no-cache-dir -r requirements.txt
 
 COPY . .
 
-RUN patch /usr/local/lib/python3.10/dist-packages/ldapdb/backends/ldap/base.py < /opt/userportal/ldapdb.patch
-RUN python3 manage.py collectstatic --noinput && python3 manage.py compilemessages
+RUN patch /opt/userportal-env/lib/python3.12/site-packages/ldapdb/backends/ldap/base.py < /opt/userportal/ldapdb.patch
+
+# Temporarily remove db version check to support mariadb server on EL8 without appstream enabled
+RUN patch /opt/userportal-env/lib/python3.12/site-packages/django/db/backends/base/base.py < /opt/userportal/dbcheck.patch
+
+RUN /opt/userportal-env/bin/python manage.py collectstatic --noinput && /opt/userportal-env/bin/python manage.py compilemessages
 
 EXPOSE 8000
 CMD ["./run-django-production.sh"]

dbcheck.patch

@@ -0,0 +1,11 @@
+--- /opt/userportal-env/lib/python3.12/site-packages/django/db/backends/base/base.py    2024-07-17 17:45:44.000000000 +0000
++++ /opt/userportal-env/lib/python3.12/site-packages/django/db/backends/base/base.py.modified   2024-07-17 19:21:39.250937980 +0000
+@@ -222,7 +222,7 @@
+         """Initialize the database connection settings."""
+         global RAN_DB_VERSION_CHECK
+         if self.alias not in RAN_DB_VERSION_CHECK:
+-            self.check_database_version_supported()
++            #self.check_database_version_supported()
+             RAN_DB_VERSION_CHECK.add(self.alias)
+
+     def create_cursor(self, name=None):

podman/deploy.sh

@@ -20,6 +20,16 @@ if [ -z "$VERSION" ]; then
     VERSION=latest
 fi
 
+if [ -z "$WORKERS" ]; then
+    echo "WORKERS is not set in the config file, using 4"
+    WORKERS=4
+fi
+
+if [ -z "$THREADS" ]; then
+    echo "THREADS is not set in the config file, using 4"
+    THREADS=4
+fi
+
 podman pod stop TT-$TT_ENV
 podman pod rm -f TT-$TT_ENV
 podman volume rm -f TT-$TT_ENV-static
@@ -28,6 +38,8 @@ podman pod create -p $PORT:80 --name TT-$TT_ENV
 podman volume create TT-$TT_ENV-static
 
 podman run --detach --pod TT-$TT_ENV --name=TT-$TT_ENV-django \
+    --env WORKERS=${WORKERS} \
+    --env THREADS=${THREADS} \
     -v $TT_ENV_PATH/99-local.py:/secrets/settings/99-local.py:Z \
     -v $TT_ENV_PATH/private.key:/opt/userportal/private.key:Z \
     -v $TT_ENV_PATH/public.cert:/opt/userportal/public.cert:Z \

requirements.txt

@@ -1,56 +1,55 @@
-asgiref==3.4.1
-certifi==2024.07.04
-cffi==1.15.1
-chardet==4.0.0
-charset-normalizer==3.3.1
-cryptography==42.0.4
-dateparser==1.0.0
+asgiref==3.8.1
+certifi==2024.7.4
+cffi==1.16.0
+charset-normalizer==3.3.2
+contourpy==1.2.1
+cryptography==42.0.8
+cycler==0.12.1
+dateparser==1.2.0
 defusedxml==0.7.1
-Django==3.2.25
-django-auth-ldap==2.3.0
-django-bootstrap-pagination==1.7.1
-django-debug-toolbar==3.2.4
-django-ldapdb==1.5.1
+Django==5.0.7
+django-bootstrap-pagination @ git+https://github.com/teejaydub/django-bootstrap-pagination@82ea7a213860a741890f324e025f6ec215f24a32
+django-crispy-forms==2.2
+django-debug-toolbar==4.4.6
+django-ldapdb @ git+https://github.com/nikolaik/django-ldapdb@381e4f40032c96ea6ae6623a5a0339b3ea526ae5
 django-settings-export==1.2.1
 django-watchman==1.3.0
-djangorestframework==3.13.1
-djangorestframework-api-key==2.0.0
-djangorestframework-datatables==0.7.0
-djangosaml2==1.5.1
-elementpath==2.4.0
-flake8==3.9.0
+django_csp==3.8
+djangorestframework==3.15.2
+djangorestframework-datatables==0.7.2
+djangosaml2==1.9.3
+elementpath==4.4.0
+flake8==7.1.0
+fonttools==4.53.1
 gunicorn==22.0.0
+httmock==1.4.0
 idna==3.7
-importlib-metadata==3.7.3
-importlib-resources==5.4.0
-install==1.3.5
-IPy==1.1
-mccabe==0.6.1
-mysqlclient==2.0.3
-numpy==1.23.4
-pandas==1.1.5
-plotly==4.14.3
-prometheus-api-client==0.4.2
-pyasn1==0.4.8
-pyasn1-modules==0.2.8
-pycodestyle==2.7.0
-pycparser==2.21
-pyflakes==2.3.0
-PyJWT==2.7.0
-Pyment==0.3.3
+kiwisolver==1.4.5
+matplotlib==3.9.1
+mccabe==0.7.0
+mysqlclient==2.2.4
+numpy==2.0.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+prometheus-api-client==0.5.5
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pycodestyle==2.12.0
+pycparser==2.22
+pyflakes==3.2.0
 pyOpenSSL==24.1.0
-pysaml2==7.1.2
-python-dateutil==2.8.1
-python-ldap==3.4.3
-pytz==2021.1
-PyYAML==6.0
-regex==2020.11.13
+pyparsing==3.1.2
+pysaml2==7.5.0
+python-dateutil==2.9.0.post0
+python-ldap==3.4.4
+pytz==2024.1
+PyYAML==6.0.1
+regex==2024.5.15
 requests==2.32.3
-retrying==1.3.3
-six==1.15.0
+six==1.16.0
 sqlparse==0.5.0
-typing-extensions==3.7.4.3
-tzlocal==2.1
-urllib3==1.26.19
-xmlschema==1.9.2
-zipp==3.19.1
+tzdata==2024.1
+tzlocal==5.2
+urllib3==2.2.2
+xmlschema==2.5.1

run-django-production.sh

@@ -7,4 +7,4 @@ set -o nounset
 cp /secrets/settings/99-local.py /opt/userportal/userportal/settings/99-local.py
 mkdir -p /var/www/api/static
 cp -r /opt/userportal/collected-static/* /var/www/api/static/
-gunicorn --bind :8000 --workers 1 --timeout 90 userportal.wsgi
+/opt/userportal-env/bin/gunicorn --bind :8000 --workers $WORKERS --threads $THREADS --timeout 90 userportal.wsgi

userportal/settings/10-base.py

@@ -1,13 +1,5 @@
 """
 Django settings for userportal project.
-
-Generated by 'django-admin startproject' using Django 3.1.7.
-
-For more information on this file, see
-https://docs.djangoproject.com/en/3.1/topics/settings/
-
-For the full list of settings and their values, see
-https://docs.djangoproject.com/en/3.1/ref/settings/
 """
 
 from pathlib import Path
@@ -43,6 +35,7 @@
     'django.contrib.staticfiles',
     'django.contrib.humanize',
     'djangosaml2',
+    'csp',
     'watchman',
 
     'pages',
@@ -78,6 +71,7 @@
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
     'djangosaml2.middleware.SamlSessionMiddleware',
+    'csp.middleware.CSPMiddleware',
 ]
 
 ROOT_URLCONF = 'userportal.urls'
@@ -204,3 +198,9 @@
     'DEFAULT_PAGINATION_CLASS': 'rest_framework_datatables.pagination.DatatablesPageNumberPagination',
     'PAGE_SIZE': 100,
 }
+
+# Content Security Policy
+CSP_DEFAULT_SRC = ("'self'")
+CSP_IMG_SRC = ("'self'", "data:", 'object-arbutus.cloud.computecanada.ca')
+CSP_STYLE_SRC = ("'self'", "'unsafe-inline'", 'cdn.jsdelivr.net', 'cdnjs.cloudflare.com', 'cdn.datatables.net')
+CSP_SCRIPT_SRC = ("'self'", "'unsafe-inline'", 'cdn.jsdelivr.net', 'cdnjs.cloudflare.com', 'cdn.datatables.net', 'code.jquery.com', 'cdn.plot.ly')