From 3cff930617eb7f383f0df1b987f5309f16cb5c07 Mon Sep 17 00:00:00 2001 From: Felix Rindt Date: Tue, 12 Sep 2023 12:12:00 +0200 Subject: [PATCH 01/14] add docker deployment --- .env.example | 9 +- Dockerfile | 32 ++ deployment/docker-compose/docker-compose.yml | 59 ++++ deployment/docker/entrypoint.sh | 21 ++ deployment/docker/ephios-docker.env | 13 + docs/admin/deployment/docker/index.rst | 4 + docs/admin/deployment/index.rst | 291 +------------------ docs/admin/deployment/manual/index.rst | 281 ++++++++++++++++++ ephios/settings.py | 136 +++++---- 9 files changed, 500 insertions(+), 346 deletions(-) create mode 100644 Dockerfile create mode 100644 deployment/docker-compose/docker-compose.yml create mode 100644 deployment/docker/entrypoint.sh create mode 100644 deployment/docker/ephios-docker.env create mode 100644 docs/admin/deployment/docker/index.rst create mode 100644 docs/admin/deployment/manual/index.rst diff --git a/.env.example b/.env.example index 7806545ab..9af6d442c 100644 --- a/.env.example +++ b/.env.example @@ -1,15 +1,8 @@ -SECRET_KEY=b0jbk58)l&^hy5cht+j$n9!laoke5ivdc+3&@qf2yrt1udvb85 DEBUG=True -DEBUG_TOOLBAR=False DATABASE_URL=sqlite:///data/db.sqlite3 ALLOWED_HOSTS="*" -STATIC_URL=/static/ -STATIC_ROOT=data/static/ -LOGGING_FILE=data/logs/ephios.log -INTERNAL_IPS="127.0.0.1" SITE_URL=http://localhost:8000 EMAIL_URL=dummymail:// DEFAULT_FROM_EMAIL=webmaster@localhost SERVER_EMAIL=root@localhost -ADMINS=Root User -# VAPID_PRIVATE_KEY_PATH=data/private_key.pem # push notifications, create in `data/` with `vapid --gen` +ADMINS="Root User " diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..ad37e8feb --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +# syntax=docker/dockerfile:1 +FROM python:3.11-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 +ENV POETRY_VIRTUALENVS_CREATE=false +ENV POETRY_VERSION=1.6.1 + +WORKDIR /app +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + gettext \ + locales && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* \ +RUN dpkg-reconfigure locales && \ + locale-gen C.UTF-8 && \ + /usr/sbin/update-locale LANG=C.UTF-8 \ +RUN mkdir -p /data/static/ +RUN pip install "poetry==$POETRY_VERSION" gunicorn + +COPY pyproject.toml /app/pyproject.toml +COPY poetry.lock /app/poetry.lock +RUN poetry install -E pgsql -E redis -E mysql +# COPY most content after poetry install to make use of caching +COPY . /app + +COPY deployment/docker/ephios-docker.env /app/.env +COPY deployment/docker/entrypoint.sh /usr/local/bin/ephios +RUN chmod +x /usr/local/bin/ephios +ENTRYPOINT ["ephios"] +CMD ["gunicorn"] \ No newline at end of file diff --git a/deployment/docker-compose/docker-compose.yml b/deployment/docker-compose/docker-compose.yml new file mode 100644 index 000000000..2b20d183f --- /dev/null +++ b/deployment/docker-compose/docker-compose.yml @@ -0,0 +1,59 @@ +services: + + ephios_django: + container_name: ephios_django + image: ghcr.io/ephios_dev/ephios:latest + restart: unless-stopped + command: gunicorn + volumes: + - ephios_data_volume:/var/ephios/data/ + depends_on: + - ephios_postgres + env_file: + - .env + networks: + - intern + hostname: django.ephios.de + + ephios_postgres: + container_name: ephios_postgres + image: postgres:14-alpine + restart: unless-stopped + volumes: + - ephios_postgres_data:/var/lib/postgresql/data/ + env_file: + - .env + environment: + - PGUSER=ephios + healthcheck: + test: [ "CMD-SHELL", "pg_isready" ] + interval: 10s + timeout: 5s + retries: 5 + networks: + - intern + + ephios_nginx: + container_name: ephios_nginx + build: ./nginx + restart: unless-stopped + volumes: + - ephios_static_volume:/app/data/static/ + - ephios_media_volume:/app/data/media/ + depends_on: + - ephios_django + networks: + - default + - intern + +volumes: + ephios_data_volume: + + +networks: + intern: + name: ephios-internal + driver: bridge + default: + name: default-network + external: true \ No newline at end of file diff --git a/deployment/docker/entrypoint.sh b/deployment/docker/entrypoint.sh new file mode 100644 index 000000000..8d382eec6 --- /dev/null +++ b/deployment/docker/entrypoint.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +NUM_WORKERS_DEFAULT=$((2 * $(nproc --all))) +export NUM_WORKERS=${NUM_WORKERS:-$NUM_WORKERS_DEFAULT} + +python manage.py migrate +python manage.py collectstatic --no-input +python manage.py compilemessages +python manage.py compilejsi18n + +echo "Starting" "$@" + +if [ "$1" == "gunicorn" ]; then + exec gunicorn ephios.wsgi \ + --name ephios \ + --workers $NUM_WORKERS \ + --max-requests 1000 \ + --max-requests-jitter 100 +fi + +exec python manage.py "$@" diff --git a/deployment/docker/ephios-docker.env b/deployment/docker/ephios-docker.env new file mode 100644 index 000000000..2862ec862 --- /dev/null +++ b/deployment/docker/ephios-docker.env @@ -0,0 +1,13 @@ +DEBUG=False +DATA_DIR=/var/ephios/data/ + +# you must set these in your deployment: + +# DATABASE_URL=sqlite:///data/db.sqlite3 +# CACHE_URL=... +# ALLOWED_HOSTS="*" +# SITE_URL=http://localhost:8000 +# EMAIL_URL=dummymail:// +# DEFAULT_FROM_EMAIL=webmaster@localhost +# SERVER_EMAIL=root@localhost +# ADMINS="Root User " diff --git a/docs/admin/deployment/docker/index.rst b/docs/admin/deployment/docker/index.rst new file mode 100644 index 000000000..117254422 --- /dev/null +++ b/docs/admin/deployment/docker/index.rst @@ -0,0 +1,4 @@ +Docker +====== + +WIP \ No newline at end of file diff --git a/docs/admin/deployment/index.rst b/docs/admin/deployment/index.rst index 303707bd2..e293c4131 100644 --- a/docs/admin/deployment/index.rst +++ b/docs/admin/deployment/index.rst @@ -3,11 +3,6 @@ Deploying ephios This section shows how to deploy ephios in a production environment. - -.. toctree:: - :maxdepth: 2 - - Prerequisites ------------- @@ -28,289 +23,15 @@ To run ephios (a django project) in production, you generally need: Installation ------------ -Generally, ephios can be installed like most django projects. We plan on providing a docker image in the future. - -Manual installation -~~~~~~~~~~~~~~~~~~~ - -To run ephios on a debian-based system, make sure you have installed the prerequisites mentioned above. -What follows is a rough guide to install ephios and configure the supporting services. -Feel free to adapt it to your needs and style. The guide assumes you are logged in as root -(``#`` is a root prompt, ``$`` the ephios user prompt). - -Unix user -''''''''' - -Create a unix user that will run the ephios. You should avoid running ephios with root privileges. - -.. code-block:: console - - # adduser --disabled-password --home /home/ephios ephios - -Package dependencies -'''''''''''''''''''' - -Install the system packages that ephios depends on. - -.. code-block:: console - - # apt-get install gettext - -python environment and ephios package -''''''''''''''''''''''''''''''''''''' - -Create a `virtualenv `_ for ephios and install the ephios package. -Replace pgsql with mysql if you want to use mysql. - -.. code-block:: console - - # sudo -u ephios python3.11 -m venv /home/ephios/venv - # sudo -u ephios /home/ephios/venv/bin/pip install gunicorn "ephios[redis,pgsql]" - -Database -'''''''' - -Create a database user and a database for ephios. For postgres this could look like this: - -.. code-block:: console - - # sudo -u postgres createuser ephios - # sudo -u postgres createdb -O ephios ephios - -Make sure the encoding of the database is UTF-8. - -Data directory -'''''''''''''' - -ephios stores some data in the file system. Create the folders and make sure the ephios user can write to them. -The reverse proxy needs to be able to read the static files stored in there. - -.. code-block:: console - - # mkdir -p /var/ephios/data/static - # chown -R ephios:ephios /var/ephios - -.. _web_push_notifications: - -VAPID keys -'''''''''' - -ephios uses `VAPID `_ to send push notifications. Create a VAPID key pair: - -.. code-block:: console - - # sudo -u ephios mkdir -p /home/ephios/vapid - # cd /home/ephios/vapid - # sudo -u ephios /home/ephios/venv/bin/vapid --gen - -Config file -''''''''''' - -ephios can be configured using environment variables. They can also be read from a file. -Create a file ``/home/ephios/ephios.env`` (owned by the ephios user) with the following -content, replacing the values with your own: - -.. code-block:: - - SECRET_KEY= - DATABASE_URL=psql://dbuser:dbpass@localhost:5432/ephios - ALLOWED_HOSTS="your.domain.org" - SITE_URL=https://your.domain.org - EMAIL_URL=smtp+ssl://emailuser:emailpass@smtp.domain.org:465 - DEFAULT_FROM_EMAIL=ephios@domain.org - SERVER_EMAIL=ephios@domain.org - ADMINS=Org Admin - VAPID_PRIVATE_KEY_PATH=/home/ephios/vapid/private_key.pem - CACHE_URL="redis://127.0.0.1:6379/1" - STATIC_ROOT=/var/ephios/data/static/ - LOGGING_FILE=/var/ephios/data/logs/ephios.log - DEBUG=False - - -For details on the configuration options and syntax, see :ref:`configuration options `. - -To test your configuration, run: - -.. code-block:: console - - # sudo -u ephios -i - $ export ENV_PATH="/home/ephios/ephios.env" - $ source /home/ephios/venv/bin/activate - $ python -m ephios check --deploy - $ python -m ephios sendtestemail --admin - -Build ephios files -'''''''''''''''''' +Generally, ephios can be installed like most django projects. +We prepared some guides for common deployment scenarios: -Now that the configuration is in place, we can build the static files and the translation files. - -.. code-block:: console - - # sudo -u ephios -i - $ export ENV_PATH="/home/ephios/ephios.env" - $ source /home/ephios/venv/bin/activate - $ python -m ephios migrate - $ python -m ephios collectstatic --noinput - $ python -m ephios compilemessages - $ python -m ephios compilejsi18n - -Setup cron -'''''''''' - -ephios needs to have the ``run_periodic`` management command run periodically (every few minutes). -This command sends notifications and performs other tasks that need to be done regularly. -Run ``crontab -e -u ephios`` and add the following line: - -.. code-block:: bash - - */5 * * * * ENV_PATH=/home/ephios/ephios.env /home/ephios/venv/bin/python -m ephios run_periodic - -Setup gunicorn systemd service -'''''''''''''''''''''''''''''' - -To run ephios with gunicorn, create a systemd service file ``/etc/systemd/system/ephios-gunicorn.service`` -with the following content: - -.. code-block:: ini - - [Unit] - Description=ephios gunicorn daemon - After=network.target - - [Service] - Type=notify - User=ephios - Group=ephios - WorkingDirectory=/home/ephios - Environment="ENV_PATH=/home/ephios/ephios.env" - ExecStart=/home/ephios/venv/bin/gunicorn ephios.wsgi --name ephios \ - --workers 5 --max-requests 1000 --max-requests-jitter 100 --bind=127.0.0.1:8327 - Restart=on-failure - - [Install] - WantedBy=multi-user.target - -To start the service run: - -.. code-block:: console - - # systemctl daemon-reload - # systemctl enable ephios-gunicorn - # systemctl start ephios-gunicorn - - -Configure reverse proxy -''''''''''''''''''''''' - -Configure your reverse proxy to forward requests to ephios. For nginx, you can start with this: - -.. code-block:: nginx - - server { - listen 80 default_server; - listen [::]:80 ipv6only=on default_server; - server_name your.domain.org - location / { - return 301 https://$host$request_uri; - } - } - - server { - listen 443 ssl; - listen [::]:443 ipv6only=on ssl; - server_name your.domain.org; - - http2 on; - ssl_certificate /etc/letsencrypt/certificates/your.domain.org.crt; - ssl_certificate_key /etc/letsencrypt/certificates/your.domain.org.key; - - location / { - proxy_pass http://localhost:8327; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto https; - proxy_set_header Host $http_host; - proxy_redirect off; - } - - location /static/ { - alias /var/ephios/data/static/; - access_log off; - expires 1d; - add_header Cache-Control "public"; - } - } - -For apache you can build on this: - -.. code-block:: apache - - - ServerName your.domain.org - Redirect permanent / https://your.domain.org/ - - - - ServerName your.domain.org - SSLEngine on - SSLCertificateFile /etc/letsencrypt/certificates/your.domain.org.crt - SSLCertificateKeyFile /etc/letsencrypt/certificates/your.domain.org.key - - ProxyPass /static/ ! - Alias /static/ /var/ephios/data/static/ - - Require all granted - - - RequestHeader set X-Forwarded-Proto "https" - ProxyPreserveHost On - ProxyPass / http://localhost:8327/ - ProxyPassReverse / http://localhost:8327/ - - -Remember to replace all the domain names and certificate paths with your own. -Make sure to use secure SSL settings. -To obtain SSL certificates, you can use `certbot `_ with Let's Encrypt. - -Next steps -'''''''''' - -After restarting your reverse proxy you should be able to access ephios at https://your.domain.org. -You can now create your first user account by running: - -.. code-block:: console - - # sudo -u ephios -i - $ export ENV_PATH="/home/ephios/ephios.env" - $ source /home/ephios/venv/bin/activate - $ python -m ephios createsuperuser - -You should now secure your installation. Try starting with the tips below. - -To install a plugin install them via pip and restart the ephios-gunicorn service: - -.. code-block:: console - - # ENV_PATH="/home/ephios/ephios.env" sudo -u ephios /home/ephios/venv/bin/pip install ephios- - # systemctl restart ephios-gunicorn - -To update ephios create a backup of your database and files and run: - -.. code-block:: console - - # sudo -u ephios -i - $ export ENV_PATH="/home/ephios/ephios.env" - $ source /home/ephios/venv/bin/activate - $ pip install -U "ephios[redis,pgsql]" - $ python -m ephios migrate - $ python -m ephios collectstatic --noinput - $ python -m ephios compilemessages - $ python -m ephios compilejsi18n - -Then, as root, restart the gunicorn service: +.. toctree:: + :maxdepth: 1 -.. code-block:: console + manual/index + docker/index - # systemctl restart ephios-gunicorn Securing your installation -------------------------- diff --git a/docs/admin/deployment/manual/index.rst b/docs/admin/deployment/manual/index.rst new file mode 100644 index 000000000..0ab331f8f --- /dev/null +++ b/docs/admin/deployment/manual/index.rst @@ -0,0 +1,281 @@ +Manual installation +~~~~~~~~~~~~~~~~~~~ + +To run ephios on a debian-based system, make sure you have prepared the prerequisites. +What follows is a rough guide to install ephios and configure the supporting services. +Feel free to adapt it to your needs and style. The guide assumes you are logged in as root +(``#`` is a root prompt, ``$`` the ephios user prompt). + +Unix user +''''''''' + +Create a unix user that will run the ephios. You should avoid running ephios with root privileges. + +.. code-block:: console + + # adduser --disabled-password --home /home/ephios ephios + +Package dependencies +'''''''''''''''''''' + +Install the system packages that ephios depends on. + +.. code-block:: console + + # apt-get install gettext + +python environment and ephios package +''''''''''''''''''''''''''''''''''''' + +Create a `virtualenv `_ for ephios and install the ephios package. +Replace pgsql with mysql if you want to use mysql. + +.. code-block:: console + + # sudo -u ephios python3.11 -m venv /home/ephios/venv + # sudo -u ephios /home/ephios/venv/bin/pip install gunicorn "ephios[redis,pgsql]" + +Database +'''''''' + +Create a database user and a database for ephios. For postgres this could look like this: + +.. code-block:: console + + # sudo -u postgres createuser ephios + # sudo -u postgres createdb -O ephios ephios + +Make sure the encoding of the database is UTF-8. + +Data directory +'''''''''''''' + +ephios stores some data in the file system. Create the folders and make sure the ephios user can write to them. +The reverse proxy needs to be able to read the static files stored in there. + +.. code-block:: console + + # mkdir -p /var/ephios/data/static + # chown -R ephios:ephios /var/ephios + +.. _web_push_notifications: + +VAPID keys +'''''''''' + +ephios uses `VAPID `_ to send push notifications. Create a VAPID key pair: + +.. code-block:: console + + # sudo -u ephios mkdir -p /home/ephios/vapid + # cd /home/ephios/vapid + # sudo -u ephios /home/ephios/venv/bin/vapid --gen + +Config file +''''''''''' + +ephios can be configured using environment variables. They can also be read from a file. +Create a file ``/home/ephios/ephios.env`` (owned by the ephios user) with the following +content, replacing the values with your own: + +.. code-block:: + + SECRET_KEY= + DATABASE_URL=psql://dbuser:dbpass@localhost:5432/ephios + ALLOWED_HOSTS="your.domain.org" + SITE_URL=https://your.domain.org + EMAIL_URL=smtp+ssl://emailuser:emailpass@smtp.domain.org:465 + DEFAULT_FROM_EMAIL=ephios@domain.org + SERVER_EMAIL=ephios@domain.org + ADMINS=Org Admin + VAPID_PRIVATE_KEY_PATH=/home/ephios/vapid/private_key.pem + CACHE_URL="redis://127.0.0.1:6379/1" + STATIC_ROOT=/var/ephios/data/static/ + LOGGING_FILE=/var/ephios/data/logs/ephios.log + DEBUG=False + + +For details on the configuration options and syntax, see :ref:`configuration options `. + +To test your configuration, run: + +.. code-block:: console + + # sudo -u ephios -i + $ export ENV_PATH="/home/ephios/ephios.env" + $ source /home/ephios/venv/bin/activate + $ python -m ephios check --deploy + $ python -m ephios sendtestemail --admin + +Build ephios files +'''''''''''''''''' + +Now that the configuration is in place, we can build the static files and the translation files. + +.. code-block:: console + + # sudo -u ephios -i + $ export ENV_PATH="/home/ephios/ephios.env" + $ source /home/ephios/venv/bin/activate + $ python -m ephios migrate + $ python -m ephios collectstatic --noinput + $ python -m ephios compilemessages + $ python -m ephios compilejsi18n + +Setup cron +'''''''''' + +ephios needs to have the ``run_periodic`` management command run periodically (every few minutes). +This command sends notifications and performs other tasks that need to be done regularly. +Run ``crontab -e -u ephios`` and add the following line: + +.. code-block:: bash + + */5 * * * * ENV_PATH=/home/ephios/ephios.env /home/ephios/venv/bin/python -m ephios run_periodic + +Setup gunicorn systemd service +'''''''''''''''''''''''''''''' + +To run ephios with gunicorn, create a systemd service file ``/etc/systemd/system/ephios-gunicorn.service`` +with the following content: + +.. code-block:: ini + + [Unit] + Description=ephios gunicorn daemon + After=network.target + + [Service] + Type=notify + User=ephios + Group=ephios + WorkingDirectory=/home/ephios + Environment="ENV_PATH=/home/ephios/ephios.env" + ExecStart=/home/ephios/venv/bin/gunicorn ephios.wsgi --name ephios \ + --workers 5 --max-requests 1000 --max-requests-jitter 100 --bind=127.0.0.1:8327 + Restart=on-failure + + [Install] + WantedBy=multi-user.target + +To start the service run: + +.. code-block:: console + + # systemctl daemon-reload + # systemctl enable ephios-gunicorn + # systemctl start ephios-gunicorn + + +Configure reverse proxy +''''''''''''''''''''''' + +Configure your reverse proxy to forward requests to ephios. For nginx, you can start with this: + +.. code-block:: nginx + + server { + listen 80 default_server; + listen [::]:80 ipv6only=on default_server; + server_name your.domain.org + location / { + return 301 https://$host$request_uri; + } + } + + server { + listen 443 ssl; + listen [::]:443 ipv6only=on ssl; + server_name your.domain.org; + + http2 on; + ssl_certificate /etc/letsencrypt/certificates/your.domain.org.crt; + ssl_certificate_key /etc/letsencrypt/certificates/your.domain.org.key; + + location / { + proxy_pass http://localhost:8327; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Host $http_host; + proxy_redirect off; + } + + location /static/ { + alias /var/ephios/data/static/; + access_log off; + expires 1d; + add_header Cache-Control "public"; + } + } + +For apache you can build on this: + +.. code-block:: apache + + + ServerName your.domain.org + Redirect permanent / https://your.domain.org/ + + + + ServerName your.domain.org + SSLEngine on + SSLCertificateFile /etc/letsencrypt/certificates/your.domain.org.crt + SSLCertificateKeyFile /etc/letsencrypt/certificates/your.domain.org.key + + ProxyPass /static/ ! + Alias /static/ /var/ephios/data/static/ + + Require all granted + + + RequestHeader set X-Forwarded-Proto "https" + ProxyPreserveHost On + ProxyPass / http://localhost:8327/ + ProxyPassReverse / http://localhost:8327/ + + +Remember to replace all the domain names and certificate paths with your own. +Make sure to use secure SSL settings. +To obtain SSL certificates, you can use `certbot `_ with Let's Encrypt. + +Next steps +'''''''''' + +After restarting your reverse proxy you should be able to access ephios at https://your.domain.org. +You can now create your first user account by running: + +.. code-block:: console + + # sudo -u ephios -i + $ export ENV_PATH="/home/ephios/ephios.env" + $ source /home/ephios/venv/bin/activate + $ python -m ephios createsuperuser + +You should now secure your installation. Try starting with the tips below. + +To install a plugin install them via pip and restart the ephios-gunicorn service: + +.. code-block:: console + + # ENV_PATH="/home/ephios/ephios.env" sudo -u ephios /home/ephios/venv/bin/pip install ephios- + # systemctl restart ephios-gunicorn + +To update ephios create a backup of your database and files and run: + +.. code-block:: console + + # sudo -u ephios -i + $ export ENV_PATH="/home/ephios/ephios.env" + $ source /home/ephios/venv/bin/activate + $ pip install -U "ephios[redis,pgsql]" + $ python -m ephios migrate + $ python -m ephios collectstatic --noinput + $ python -m ephios compilemessages + $ python -m ephios compilejsi18n + +Then, as root, restart the gunicorn service: + +.. code-block:: console + + # systemctl restart ephios-gunicorn \ No newline at end of file diff --git a/ephios/settings.py b/ephios/settings.py index 210c21fd4..592c20002 100644 --- a/ephios/settings.py +++ b/ephios/settings.py @@ -1,6 +1,7 @@ import copy import datetime import os +import string from datetime import timedelta from email.utils import getaddresses from pathlib import Path @@ -8,6 +9,7 @@ import environ from cryptography.hazmat.primitives import serialization from django.contrib.messages import constants +from django.utils.crypto import get_random_string from django.utils.translation import gettext_lazy from py_vapid import Vapid, b64urlencode @@ -16,8 +18,10 @@ except ImportError: import importlib.metadata as importlib_metadata +# BASE_DIR is the directory containing the ephios package BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + env = environ.Env() # for syntax see https://django-environ.readthedocs.io/en/latest/ # read env file from ENV_PATH or fall back to a .env file in the project root @@ -25,14 +29,41 @@ print(f"Loading ephios environment from {Path(env_path).absolute()}") environ.Env.read_env(env_file=env_path) -SECRET_KEY = env.str("SECRET_KEY") DEBUG = env.bool("DEBUG") -ALLOWED_HOSTS = env.list("ALLOWED_HOSTS") +if DEBUG: + DATA_DIR = env.str("DATA_DIR", os.path.join(BASE_DIR, "data")) +else: + # DATA_DIR must be set explicitly in production + DATA_DIR = env.str("DATA_DIR") + +LOG_DIR = env.str("LOG_DIR", os.path.join(DATA_DIR, "logs")) +MEDIA_ROOT = env.str("MEDIA_ROOT", os.path.join(DATA_DIR, "media")) +STATIC_ROOT = env.str("STATIC_ROOT", os.path.join(DATA_DIR, "static")) + +for path in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT]: + if not os.path.exists(path): + os.makedirs(path, exist_ok=True) + +if "SECRET_KEY" in env: + SECRET_KEY = env.str("SECRET_KEY") +else: + SECRET_FILE = os.path.join(DATA_DIR, ".secret") + if os.path.exists(SECRET_FILE): + with open(SECRET_FILE, "r") as f: + SECRET_KEY = f.read().strip() + else: + chars = string.ascii_letters + string.digits + string.punctuation + SECRET_KEY = get_random_string(50, chars) + with open(SECRET_FILE, "w") as f: + os.chmod(SECRET_FILE, 0o600) + try: + os.chown(SECRET_FILE, os.getuid(), os.getgid()) + except AttributeError: + pass # os.chown is not available on Windows + f.write(SECRET_KEY) -data_dir = os.path.join(BASE_DIR, "data") -if DEBUG and not os.path.exists(data_dir): - # create data dir for development - os.mkdir(data_dir) + +ALLOWED_HOSTS = env.list("ALLOWED_HOSTS") try: EPHIOS_VERSION = importlib_metadata.version("ephios") @@ -40,20 +71,6 @@ # ephios is not installed as a package (e.g. development setup) EPHIOS_VERSION = "dev" -if not DEBUG: - SESSION_COOKIE_SECURE = True - CSRF_COOKIE_SECURE = True - X_FRAME_OPTIONS = "DENY" - SECURE_CONTENT_TYPE_NOSNIFF = True - SECURE_SSL_REDIRECT = True - SECURE_REFERRER_POLICY = "same-origin" - # 1 day by default, change to 1 year in production (see deployment docs) - SECURE_HSTS_SECONDS = env.int("SECURE_HSTS_SECONDS", default=3600 * 24) - SECURE_HSTS_INCLUDE_SUBDOMAINS = env.bool("SECURE_HSTS_INCLUDE_SUBDOMAINS", default=False) - SECURE_HSTS_PRELOAD = env.bool("SECURE_HSTS_PRELOAD", default=False) - - CONN_MAX_AGE = env.int("CONN_MAX_AGE", default=0) - INSTALLED_APPS = [ # we need to import our own modules before everything else to allow template # customizing e.g. for django-oauth-toolkit @@ -207,10 +224,6 @@ STATIC_URL = env.str("STATIC_URL", default="/static/") -STATIC_ROOT = env.str("STATIC_ROOT") -if not os.path.isabs(STATIC_ROOT): - STATIC_ROOT = os.path.join(BASE_DIR, STATIC_ROOT) - STATICFILES_DIRS = (os.path.join(BASE_DIR, "ephios/static"),) STATICFILES_FINDERS = ( "django.contrib.staticfiles.finders.FileSystemFinder", @@ -229,11 +242,6 @@ ADMINS = getaddresses([env("ADMINS")]) # logging -LOGGING_FILE = env.str("LOGGING_FILE", default=None) -use_file_logging = not DEBUG and LOGGING_FILE is not None -if use_file_logging: - Path(LOGGING_FILE).parent.mkdir(parents=True, exist_ok=True) - LOGGING = { "version": 1, "disable_existing_loggers": False, @@ -267,16 +275,12 @@ **( { "class": "logging.handlers.TimedRotatingFileHandler", - "filename": LOGGING_FILE, + "filename": os.path.join(LOG_DIR, "ephios.log"), "when": "midnight", "backupCount": env.int("LOGGING_BACKUP_DAYS", default=14), "atTime": datetime.time(4), "encoding": "utf-8", } - if use_file_logging - else { - "class": "logging.NullHandler", - } ), }, }, @@ -303,6 +307,14 @@ }, } + +def GET_SITE_URL(): + site_url = env.str("SITE_URL") + if site_url.endswith("/"): + site_url = site_url[:-1] + return site_url + + # Guardian configuration ANONYMOUS_USER_NAME = None GUARDIAN_MONKEY_PATCH = False @@ -321,7 +333,7 @@ INSTALLED_APPS.append("django_extensions") INSTALLED_APPS.append("debug_toolbar") MIDDLEWARE.insert(0, "debug_toolbar.middleware.DebugToolbarMiddleware") - INTERNAL_IPS = env.str("INTERNAL_IPS") + INTERNAL_IPS = env.list("INTERNAL_IPS", default=["127.0.0.1"]) # django-csp # Bootstrap requires embedded SVG files loaded via a data URI. This is not ideal, but will only be fixed in @@ -374,24 +386,23 @@ ] # django-webpush -if vapid_private_key_path := env.str("VAPID_PRIVATE_KEY_PATH", None): - vp = Vapid().from_file(vapid_private_key_path) - WEBPUSH_SETTINGS = { - "VAPID_PUBLIC_KEY": b64urlencode( - vp.public_key.public_bytes( - serialization.Encoding.X962, serialization.PublicFormat.UncompressedPoint - ) - ), - "VAPID_PRIVATE_KEY": vp, - "VAPID_ADMIN_EMAIL": ADMINS[0][1], - } - - -def GET_SITE_URL(): - site_url = env.str("SITE_URL") - if site_url.endswith("/"): - site_url = site_url[:-1] - return site_url +VAPID_PRIVATE_KEY_PATH = env.str("VAPID_PRIVATE_KEY_PATH", os.path.join(DATA_DIR, "vapid_key.pem")) +if not os.path.exists(VAPID_PRIVATE_KEY_PATH): + vapid = Vapid() + vapid.generate_keys() + vapid.save_key(VAPID_PRIVATE_KEY_PATH) + vapid.save_public_key(VAPID_PRIVATE_KEY_PATH + ".pub") + +vp = Vapid().from_file(VAPID_PRIVATE_KEY_PATH) +WEBPUSH_SETTINGS = { + "VAPID_PUBLIC_KEY": b64urlencode( + vp.public_key.public_bytes( + serialization.Encoding.X962, serialization.PublicFormat.UncompressedPoint + ) + ), + "VAPID_PRIVATE_KEY": vp, + "VAPID_ADMIN_EMAIL": ADMINS[0][1], +} DEFAULT_LISTVIEW_PAGINATION = 100 @@ -431,6 +442,25 @@ def GET_SITE_URL(): "REFRESH_TOKEN_EXPIRE_SECONDS": timedelta(days=90), } +""" +SECURITY SETTINGS +""" +if not DEBUG: + SESSION_COOKIE_SECURE = True + CSRF_COOKIE_SECURE = True + X_FRAME_OPTIONS = "DENY" + SECURE_CONTENT_TYPE_NOSNIFF = True + SECURE_SSL_REDIRECT = True + SECURE_REFERRER_POLICY = "same-origin" + # 1 day by default, change to 1 year in production (see deployment docs) + SECURE_HSTS_SECONDS = env.int("SECURE_HSTS_SECONDS", default=3600 * 24) + SECURE_HSTS_INCLUDE_SUBDOMAINS = env.bool("SECURE_HSTS_INCLUDE_SUBDOMAINS", default=False) + SECURE_HSTS_PRELOAD = env.bool("SECURE_HSTS_PRELOAD", default=False) + CONN_MAX_AGE = env.int("CONN_MAX_AGE", default=0) + +""" +OIDC SETTINGS +""" if ENABLE_OIDC_CLIENT := env.bool("ENABLE_OIDC_CLIENT", False): INSTALLED_APPS.append("mozilla_django_oidc") AUTHENTICATION_BACKENDS.append("ephios.extra.auth.EphiosOIDCAB") From c7527d2e6d3c7a887f7251e1716f4d71cee3e2a6 Mon Sep 17 00:00:00 2001 From: Felix Rindt Date: Tue, 12 Sep 2023 17:49:51 +0200 Subject: [PATCH 02/14] docker-compose --- .dockerignore | 144 ++++++++++++++++++ Dockerfile | 30 ++-- deployment/docker-compose/docker-compose.yml | 59 ------- deployment/docker/cron.sh | 7 + deployment/docker/cronjob | 4 + deployment/docker/docker-compose.yaml | 38 +++++ deployment/docker/entrypoint.sh | 24 +-- deployment/docker/ephios-docker-defaults.env | 9 ++ deployment/docker/ephios-docker.env | 13 -- deployment/docker/nginx.conf | 19 +++ deployment/docker/supervisord.conf | 23 +++ .../extra/management/commands/run_periodic.py | 5 + ephios/settings.py | 2 +- poetry.lock | 17 ++- pyproject.toml | 2 +- 15 files changed, 288 insertions(+), 108 deletions(-) create mode 100644 .dockerignore delete mode 100644 deployment/docker-compose/docker-compose.yml create mode 100644 deployment/docker/cron.sh create mode 100644 deployment/docker/cronjob create mode 100644 deployment/docker/docker-compose.yaml create mode 100644 deployment/docker/ephios-docker-defaults.env delete mode 100644 deployment/docker/ephios-docker.env create mode 100644 deployment/docker/nginx.conf create mode 100644 deployment/docker/supervisord.conf diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..8978e1860 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,144 @@ +# not execution related files +docs +tests + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# build artifacts +docs/api/ephios-open-api-schema.yml + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +data + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# IDEA files +.idea/ diff --git a/Dockerfile b/Dockerfile index ad37e8feb..b0282b2eb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,27 +6,37 @@ ENV PYTHONUNBUFFERED=1 ENV POETRY_VIRTUALENVS_CREATE=false ENV POETRY_VERSION=1.6.1 -WORKDIR /app +WORKDIR /usr/src/ephios + RUN apt-get update && \ apt-get install -y --no-install-recommends \ gettext \ + cron \ + supervisor \ locales && \ apt-get clean && \ - rm -rf /var/lib/apt/lists/* \ + rm -rf /var/lib/apt/lists/* + RUN dpkg-reconfigure locales && \ locale-gen C.UTF-8 && \ - /usr/sbin/update-locale LANG=C.UTF-8 \ -RUN mkdir -p /data/static/ + /usr/sbin/update-locale LANG=C.UTF-8 + RUN pip install "poetry==$POETRY_VERSION" gunicorn -COPY pyproject.toml /app/pyproject.toml -COPY poetry.lock /app/poetry.lock +RUN mkdir -p /var/ephios/data/ && \ + mkdir -p /var/log/supervisord/ && \ + mkdir -p /var/run/supervisord/ + +COPY . /usr/src/ephios RUN poetry install -E pgsql -E redis -E mysql -# COPY most content after poetry install to make use of caching -COPY . /app -COPY deployment/docker/ephios-docker.env /app/.env COPY deployment/docker/entrypoint.sh /usr/local/bin/ephios RUN chmod +x /usr/local/bin/ephios + +COPY deployment/docker/cronjob /etc/cron.d/ephios-cron +COPY deployment/docker/supervisord.conf /etc/supervisord.conf +COPY deployment/docker/cron.sh /usr/local/bin/cron.sh +RUN chmod +x /usr/local/bin/cron.sh + ENTRYPOINT ["ephios"] -CMD ["gunicorn"] \ No newline at end of file +CMD ["run"] \ No newline at end of file diff --git a/deployment/docker-compose/docker-compose.yml b/deployment/docker-compose/docker-compose.yml deleted file mode 100644 index 2b20d183f..000000000 --- a/deployment/docker-compose/docker-compose.yml +++ /dev/null @@ -1,59 +0,0 @@ -services: - - ephios_django: - container_name: ephios_django - image: ghcr.io/ephios_dev/ephios:latest - restart: unless-stopped - command: gunicorn - volumes: - - ephios_data_volume:/var/ephios/data/ - depends_on: - - ephios_postgres - env_file: - - .env - networks: - - intern - hostname: django.ephios.de - - ephios_postgres: - container_name: ephios_postgres - image: postgres:14-alpine - restart: unless-stopped - volumes: - - ephios_postgres_data:/var/lib/postgresql/data/ - env_file: - - .env - environment: - - PGUSER=ephios - healthcheck: - test: [ "CMD-SHELL", "pg_isready" ] - interval: 10s - timeout: 5s - retries: 5 - networks: - - intern - - ephios_nginx: - container_name: ephios_nginx - build: ./nginx - restart: unless-stopped - volumes: - - ephios_static_volume:/app/data/static/ - - ephios_media_volume:/app/data/media/ - depends_on: - - ephios_django - networks: - - default - - intern - -volumes: - ephios_data_volume: - - -networks: - intern: - name: ephios-internal - driver: bridge - default: - name: default-network - external: true \ No newline at end of file diff --git a/deployment/docker/cron.sh b/deployment/docker/cron.sh new file mode 100644 index 000000000..394c8354b --- /dev/null +++ b/deployment/docker/cron.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +while [ true ]; do + echo "Running cron job" + /usr/local/bin/python3 -m ephios run_periodic + sleep 60 +done \ No newline at end of file diff --git a/deployment/docker/cronjob b/deployment/docker/cronjob new file mode 100644 index 000000000..f5844b337 --- /dev/null +++ b/deployment/docker/cronjob @@ -0,0 +1,4 @@ +SHELL=/bin/bash +PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin +*/1 * * * * root /usr/local/bin/python -m ephios run_periodic +*/1 * * * * root /usr/bin/touch /$(/usr/bin/date +%s).txt diff --git a/deployment/docker/docker-compose.yaml b/deployment/docker/docker-compose.yaml new file mode 100644 index 000000000..cebd51b34 --- /dev/null +++ b/deployment/docker/docker-compose.yaml @@ -0,0 +1,38 @@ +services: + ephios: + build: ../../ + restart: unless-stopped + environment: + - EMAIL_URL=consolemail:// + env_file: + - ephios-docker-defaults.env + ports: + - 8000:80 + volumes: + - ephios_django_data:/var/ephios/data/ + + ephios_postgres: + image: postgres:12 + restart: unless-stopped + environment: + POSTGRES_DB: ephios + POSTGRES_USER: ephios + POSTGRES_PASSWORD: ephios + volumes: + - ephios_postgres_data:/var/lib/postgresql/data + + ephios_nginx: + image: nginx:1.19 + restart: unless-stopped + ports: + - 80:80 + - 443:443 + volumes: + - ./nginx.conf:/etc/nginx/conf.d/default.conf + - ephios_django_data:/var/ephios/data/ + depends_on: + - ephios + +volumes: + ephios_django_data: { } + ephios_postgres_data: { } diff --git a/deployment/docker/entrypoint.sh b/deployment/docker/entrypoint.sh index 8d382eec6..325e425bc 100644 --- a/deployment/docker/entrypoint.sh +++ b/deployment/docker/entrypoint.sh @@ -1,21 +1,13 @@ #!/bin/bash -NUM_WORKERS_DEFAULT=$((2 * $(nproc --all))) -export NUM_WORKERS=${NUM_WORKERS:-$NUM_WORKERS_DEFAULT} - -python manage.py migrate -python manage.py collectstatic --no-input -python manage.py compilemessages -python manage.py compilejsi18n - -echo "Starting" "$@" - -if [ "$1" == "gunicorn" ]; then - exec gunicorn ephios.wsgi \ - --name ephios \ - --workers $NUM_WORKERS \ - --max-requests 1000 \ - --max-requests-jitter 100 +set -e + +if [ "$1" == "run" ]; then + python manage.py migrate + python manage.py collectstatic --no-input + python manage.py compilemessages + python manage.py compilejsi18n + exec supervisord -n -c /etc/supervisord.conf fi exec python manage.py "$@" diff --git a/deployment/docker/ephios-docker-defaults.env b/deployment/docker/ephios-docker-defaults.env new file mode 100644 index 000000000..3cbba66a5 --- /dev/null +++ b/deployment/docker/ephios-docker-defaults.env @@ -0,0 +1,9 @@ +DEBUG=True +DATA_DIR=/var/ephios/data/ +DATABASE_URL=postgres://ephios:ephios@ephios_postgres/ephios +ALLOWED_HOSTS="*" +SITE_URL=http://localhost +EMAIL_URL=dummymail:// +DEFAULT_FROM_EMAIL=webmaster@localhost +SERVER_EMAIL=root@localhost +ADMINS="Root User " \ No newline at end of file diff --git a/deployment/docker/ephios-docker.env b/deployment/docker/ephios-docker.env deleted file mode 100644 index 2862ec862..000000000 --- a/deployment/docker/ephios-docker.env +++ /dev/null @@ -1,13 +0,0 @@ -DEBUG=False -DATA_DIR=/var/ephios/data/ - -# you must set these in your deployment: - -# DATABASE_URL=sqlite:///data/db.sqlite3 -# CACHE_URL=... -# ALLOWED_HOSTS="*" -# SITE_URL=http://localhost:8000 -# EMAIL_URL=dummymail:// -# DEFAULT_FROM_EMAIL=webmaster@localhost -# SERVER_EMAIL=root@localhost -# ADMINS="Root User " diff --git a/deployment/docker/nginx.conf b/deployment/docker/nginx.conf new file mode 100644 index 000000000..90a575267 --- /dev/null +++ b/deployment/docker/nginx.conf @@ -0,0 +1,19 @@ +server { + listen 80 default_server; + listen [::]:80 ipv6only=on default_server; + + location / { + proxy_pass http://ephios:80; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Host $http_host; + proxy_redirect off; + } + + location /static/ { + alias /var/ephios/data/static/; + access_log off; + expires 1d; + add_header Cache-Control "public"; + } +} diff --git a/deployment/docker/supervisord.conf b/deployment/docker/supervisord.conf new file mode 100644 index 000000000..dfcc8e9ae --- /dev/null +++ b/deployment/docker/supervisord.conf @@ -0,0 +1,23 @@ +[supervisord] +nodaemon=true +user=root +logfile=/var/log/supervisord/supervisord.log +pidfile=/var/run/supervisord/supervisord.pid +childlogdir=/var/log/supervisord/ +logfile_maxbytes=50MB +logfile_backups=10 +loglevel=error + +[program:gunicorn] +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 +command=gunicorn ephios.wsgi --bind 0.0.0.0:80 --name ephios --workers 5 --max-requests 1000 --max-requests-jitter 100 + +[program:cronscript] +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 +command=/usr/local/bin/cron.sh diff --git a/ephios/extra/management/commands/run_periodic.py b/ephios/extra/management/commands/run_periodic.py index 23753ce73..01b00965f 100644 --- a/ephios/extra/management/commands/run_periodic.py +++ b/ephios/extra/management/commands/run_periodic.py @@ -1,8 +1,13 @@ +import logging + from django.core.management import BaseCommand from ephios.core.signals import periodic_signal +logger = logging.getLogger(__name__) + class Command(BaseCommand): def handle(self, *args, **options): + logger.info("Running periodic tasks") periodic_signal.send(self) diff --git a/ephios/settings.py b/ephios/settings.py index 592c20002..bdc82a3b5 100644 --- a/ephios/settings.py +++ b/ephios/settings.py @@ -271,7 +271,7 @@ "file": { "level": "DEBUG", "formatter": "default", - "filters": ["require_debug_false"], + "filters": [], **( { "class": "logging.handlers.TimedRotatingFileHandler", diff --git a/poetry.lock b/poetry.lock index 9022d834f..ddafa7400 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2491,19 +2491,20 @@ files = [ [[package]] name = "urllib3" -version = "1.26.16" +version = "2.0.4" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +python-versions = ">=3.7" files = [ - {file = "urllib3-1.26.16-py2.py3-none-any.whl", hash = "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f"}, - {file = "urllib3-1.26.16.tar.gz", hash = "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14"}, + {file = "urllib3-2.0.4-py3-none-any.whl", hash = "sha256:de7df1803967d2c2a98e4b11bb7d6bd9210474c46e8a0401514e3a42a75ebde4"}, + {file = "urllib3-2.0.4.tar.gz", hash = "sha256:8d22f86aae8ef5e410d4f539fde9ce6b2113a001bb4d189e0aed70642d602b11"}, ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] -socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] [[package]] name = "virtualenv" @@ -2693,4 +2694,4 @@ redis = ["redis"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "19fe437bbd665faddf5161182386e503e31081c4d66f7dbb170644a51cc4a710" +content-hash = "e7c2bba0b7fb7204afb028fa569e759f95b52d0f8edde6e736dede15d82baa8e" diff --git a/pyproject.toml b/pyproject.toml index e33364903..282bb8635 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ uritemplate = "^4.1.1" django-recurrence = "^1.11.1" django-oauth-toolkit = "^2.3.0" mozilla-django-oidc = "^3.0.0" -urllib3 = "^1.26.0,<2.0.0" # pinned because of poetry issues with urllib3 2.0.0 +urllib3 = "^2.0.4" pyyaml = "^6.0.1" lxml = "^4.9.3" beautifulsoup4 = "^4.12.2" From 53ea63aafca01d44fd02341c1689df69f6fac3e9 Mon Sep 17 00:00:00 2001 From: Felix Rindt Date: Tue, 12 Sep 2023 18:29:57 +0200 Subject: [PATCH 03/14] secure --- Dockerfile | 4 ++++ deployment/docker/docker-compose.yaml | 5 +++++ deployment/docker/ephios-docker-defaults.env | 4 +++- deployment/docker/nginx.conf | 2 +- ephios/settings.py | 3 +++ 5 files changed, 16 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b0282b2eb..34e913134 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,11 +22,15 @@ RUN dpkg-reconfigure locales && \ /usr/sbin/update-locale LANG=C.UTF-8 RUN pip install "poetry==$POETRY_VERSION" gunicorn +RUN poetry self add "poetry-dynamic-versioning[plugin]" RUN mkdir -p /var/ephios/data/ && \ mkdir -p /var/log/supervisord/ && \ mkdir -p /var/run/supervisord/ +#COPY pyproject.toml poetry.lock .git /usr/src/ephios/ +#RUN poetry install -E pgsql -E redis -E mysql +# good caching point COPY . /usr/src/ephios RUN poetry install -E pgsql -E redis -E mysql diff --git a/deployment/docker/docker-compose.yaml b/deployment/docker/docker-compose.yaml index cebd51b34..ea5de7b10 100644 --- a/deployment/docker/docker-compose.yaml +++ b/deployment/docker/docker-compose.yaml @@ -11,6 +11,11 @@ services: volumes: - ephios_django_data:/var/ephios/data/ + ephios_redis: + image: redis:7-alpine + command: redis-server + restart: unless-stopped + ephios_postgres: image: postgres:12 restart: unless-stopped diff --git a/deployment/docker/ephios-docker-defaults.env b/deployment/docker/ephios-docker-defaults.env index 3cbba66a5..2718e2a1b 100644 --- a/deployment/docker/ephios-docker-defaults.env +++ b/deployment/docker/ephios-docker-defaults.env @@ -1,6 +1,8 @@ -DEBUG=True +DEBUG=False +TRUST_X_FORWARDED_PROTO=True DATA_DIR=/var/ephios/data/ DATABASE_URL=postgres://ephios:ephios@ephios_postgres/ephios +CACHE_URL=redis://ephios_redis/0 ALLOWED_HOSTS="*" SITE_URL=http://localhost EMAIL_URL=dummymail:// diff --git a/deployment/docker/nginx.conf b/deployment/docker/nginx.conf index 90a575267..416a1f6ef 100644 --- a/deployment/docker/nginx.conf +++ b/deployment/docker/nginx.conf @@ -5,7 +5,7 @@ server { location / { proxy_pass http://ephios:80; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Proto https; # TODO change to $proto proxy_set_header Host $http_host; proxy_redirect off; } diff --git a/ephios/settings.py b/ephios/settings.py index bdc82a3b5..423527e70 100644 --- a/ephios/settings.py +++ b/ephios/settings.py @@ -458,6 +458,9 @@ def GET_SITE_URL(): SECURE_HSTS_PRELOAD = env.bool("SECURE_HSTS_PRELOAD", default=False) CONN_MAX_AGE = env.int("CONN_MAX_AGE", default=0) + if env.bool("TRUST_X_FORWARDED_PROTO", default=False): + SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") + """ OIDC SETTINGS """ From fe86eac98f0d1e0f0a49a3cdf68faf746c6296d6 Mon Sep 17 00:00:00 2001 From: Felix Rindt Date: Wed, 13 Sep 2023 00:32:16 +0200 Subject: [PATCH 04/14] move env vars to docker-compose.yaml --- Dockerfile | 2 -- deployment/docker/cron.sh | 2 +- deployment/docker/cronjob | 4 ---- deployment/docker/docker-compose.yaml | 18 ++++++++++++------ deployment/docker/ephios-docker-defaults.env | 11 ----------- deployment/docker/nginx.conf | 2 +- ephios/settings.py | 4 ++-- 7 files changed, 16 insertions(+), 27 deletions(-) delete mode 100644 deployment/docker/cronjob delete mode 100644 deployment/docker/ephios-docker-defaults.env diff --git a/Dockerfile b/Dockerfile index 34e913134..d45cad8f6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,6 @@ WORKDIR /usr/src/ephios RUN apt-get update && \ apt-get install -y --no-install-recommends \ gettext \ - cron \ supervisor \ locales && \ apt-get clean && \ @@ -37,7 +36,6 @@ RUN poetry install -E pgsql -E redis -E mysql COPY deployment/docker/entrypoint.sh /usr/local/bin/ephios RUN chmod +x /usr/local/bin/ephios -COPY deployment/docker/cronjob /etc/cron.d/ephios-cron COPY deployment/docker/supervisord.conf /etc/supervisord.conf COPY deployment/docker/cron.sh /usr/local/bin/cron.sh RUN chmod +x /usr/local/bin/cron.sh diff --git a/deployment/docker/cron.sh b/deployment/docker/cron.sh index 394c8354b..a6492d73b 100644 --- a/deployment/docker/cron.sh +++ b/deployment/docker/cron.sh @@ -1,7 +1,7 @@ #!/bin/bash while [ true ]; do + sleep 60 echo "Running cron job" /usr/local/bin/python3 -m ephios run_periodic - sleep 60 done \ No newline at end of file diff --git a/deployment/docker/cronjob b/deployment/docker/cronjob deleted file mode 100644 index f5844b337..000000000 --- a/deployment/docker/cronjob +++ /dev/null @@ -1,4 +0,0 @@ -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -*/1 * * * * root /usr/local/bin/python -m ephios run_periodic -*/1 * * * * root /usr/bin/touch /$(/usr/bin/date +%s).txt diff --git a/deployment/docker/docker-compose.yaml b/deployment/docker/docker-compose.yaml index ea5de7b10..94f176302 100644 --- a/deployment/docker/docker-compose.yaml +++ b/deployment/docker/docker-compose.yaml @@ -3,11 +3,18 @@ services: build: ../../ restart: unless-stopped environment: - - EMAIL_URL=consolemail:// - env_file: - - ephios-docker-defaults.env - ports: - - 8000:80 + DEBUG: "False" + TRUST_X_FORWARDED_PROTO: "True" + DATA_DIR: "/var/ephios/data/" + DATABASE_URL: "postgres://ephios:ephios@ephios_postgres/ephios" + CACHE_URL: "redis://ephios_redis/0" + ALLOWED_HOSTS: "*" + # change the following to your needs + EMAIL_URL: "consolemail://" + SITE_URL: "http://localhost" + DEFAULT_FROM_EMAIL: "webmaster@localhost" + SERVER_EMAIL: "root@localhost" + ADMINS: "Root User " volumes: - ephios_django_data:/var/ephios/data/ @@ -31,7 +38,6 @@ services: restart: unless-stopped ports: - 80:80 - - 443:443 volumes: - ./nginx.conf:/etc/nginx/conf.d/default.conf - ephios_django_data:/var/ephios/data/ diff --git a/deployment/docker/ephios-docker-defaults.env b/deployment/docker/ephios-docker-defaults.env deleted file mode 100644 index 2718e2a1b..000000000 --- a/deployment/docker/ephios-docker-defaults.env +++ /dev/null @@ -1,11 +0,0 @@ -DEBUG=False -TRUST_X_FORWARDED_PROTO=True -DATA_DIR=/var/ephios/data/ -DATABASE_URL=postgres://ephios:ephios@ephios_postgres/ephios -CACHE_URL=redis://ephios_redis/0 -ALLOWED_HOSTS="*" -SITE_URL=http://localhost -EMAIL_URL=dummymail:// -DEFAULT_FROM_EMAIL=webmaster@localhost -SERVER_EMAIL=root@localhost -ADMINS="Root User " \ No newline at end of file diff --git a/deployment/docker/nginx.conf b/deployment/docker/nginx.conf index 416a1f6ef..90a575267 100644 --- a/deployment/docker/nginx.conf +++ b/deployment/docker/nginx.conf @@ -5,7 +5,7 @@ server { location / { proxy_pass http://ephios:80; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto https; # TODO change to $proto + proxy_set_header X-Forwarded-Proto https; proxy_set_header Host $http_host; proxy_redirect off; } diff --git a/ephios/settings.py b/ephios/settings.py index 423527e70..5bafad409 100644 --- a/ephios/settings.py +++ b/ephios/settings.py @@ -458,8 +458,8 @@ def GET_SITE_URL(): SECURE_HSTS_PRELOAD = env.bool("SECURE_HSTS_PRELOAD", default=False) CONN_MAX_AGE = env.int("CONN_MAX_AGE", default=0) - if env.bool("TRUST_X_FORWARDED_PROTO", default=False): - SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") +if env.bool("TRUST_X_FORWARDED_PROTO", default=False): + SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") """ OIDC SETTINGS From 1abfce6f40962f2f1b8040f02fb714d37300caf8 Mon Sep 17 00:00:00 2001 From: Felix Rindt Date: Wed, 13 Sep 2023 10:15:37 +0200 Subject: [PATCH 05/14] add docker documentation --- .env.example | 1 + .../{docker => compose}/docker-compose.yaml | 0 deployment/{docker => compose}/nginx.conf | 0 docs/admin/deployment/docker/index.rst | 52 ++++++++++++++++++- docs/admin/deployment/manual/index.rst | 17 ++---- docs/development/contributing.rst | 2 - ephios/settings.py | 6 +-- 7 files changed, 58 insertions(+), 20 deletions(-) rename deployment/{docker => compose}/docker-compose.yaml (100%) rename deployment/{docker => compose}/nginx.conf (100%) diff --git a/.env.example b/.env.example index 9af6d442c..40512c5ff 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,5 @@ DEBUG=True +DATA_DIR=data DATABASE_URL=sqlite:///data/db.sqlite3 ALLOWED_HOSTS="*" SITE_URL=http://localhost:8000 diff --git a/deployment/docker/docker-compose.yaml b/deployment/compose/docker-compose.yaml similarity index 100% rename from deployment/docker/docker-compose.yaml rename to deployment/compose/docker-compose.yaml diff --git a/deployment/docker/nginx.conf b/deployment/compose/nginx.conf similarity index 100% rename from deployment/docker/nginx.conf rename to deployment/compose/nginx.conf diff --git a/docs/admin/deployment/docker/index.rst b/docs/admin/deployment/docker/index.rst index 117254422..1d588ffb3 100644 --- a/docs/admin/deployment/docker/index.rst +++ b/docs/admin/deployment/docker/index.rst @@ -1,4 +1,54 @@ Docker ====== -WIP \ No newline at end of file +Docker Image +------------ + +We automatically build a Docker image for every release. +It uses gunicorn as a WSGI-server and still requires a +database, redis and some ssl-terminating webserver/proxy +to be set up. + +The image is available on the Github Container Registry. + +Deployment with Docker Compose +------------------------------ + +We provide a basic docker-compose file that adds +nginx, postgres and redis to the mix and exposes +the app on port 80. You still need to either provide +an SSL-terminating proxy in front of it or +configure the nginx container to do so. + +The compose file can be found at +``deployment/compose/docker-compose.yml`` and +`on github `_. +Feel free to use it as a starting point for your own deployment. + +The container defines two volumes for the database and +ephios data files. You should mount them to a persistent +location on your host. + +Make sure to change the environment variables in the +compose file to your needs. Have a look at +the :ref:`configuration options `. + +To start the compose file and add a first superuser run: + +.. code-block:: console + + # cd deployment/compose/ + # docker compose up --build + # docker exec -it compose-ephios-1 python -m ephios createsuperuser + +If you want to test the container without https, that's +only possible by changing it the compose file: + +.. code-block:: yaml + + DEBUG: "True" + TRUST_X_FORWARDED_PROTO: "False" + +The first line enables debug mode and therefore disables a redirect from http to https. +The second line disables trusting the X-Forwarded-Proto header set to https by the nginx +container, which is required for djangos CSRF protection to not kick in on http requests. \ No newline at end of file diff --git a/docs/admin/deployment/manual/index.rst b/docs/admin/deployment/manual/index.rst index 0ab331f8f..8b596c16b 100644 --- a/docs/admin/deployment/manual/index.rst +++ b/docs/admin/deployment/manual/index.rst @@ -63,13 +63,9 @@ The reverse proxy needs to be able to read the static files stored in there. VAPID keys '''''''''' -ephios uses `VAPID `_ to send push notifications. Create a VAPID key pair: - -.. code-block:: console - - # sudo -u ephios mkdir -p /home/ephios/vapid - # cd /home/ephios/vapid - # sudo -u ephios /home/ephios/venv/bin/vapid --gen +ephios uses `VAPID `_ to send push notifications. +Vapid keys will be generated automatically if they are not present. +If you want to use your own keys, you can generate them with ``/home/ephios/venv/bin/vapid --gen``. Config file ''''''''''' @@ -80,7 +76,8 @@ content, replacing the values with your own: .. code-block:: - SECRET_KEY= + DEBUG=False + DATA_DIR=/var/ephios/data DATABASE_URL=psql://dbuser:dbpass@localhost:5432/ephios ALLOWED_HOSTS="your.domain.org" SITE_URL=https://your.domain.org @@ -88,11 +85,7 @@ content, replacing the values with your own: DEFAULT_FROM_EMAIL=ephios@domain.org SERVER_EMAIL=ephios@domain.org ADMINS=Org Admin - VAPID_PRIVATE_KEY_PATH=/home/ephios/vapid/private_key.pem CACHE_URL="redis://127.0.0.1:6379/1" - STATIC_ROOT=/var/ephios/data/static/ - LOGGING_FILE=/var/ephios/data/logs/ephios.log - DEBUG=False For details on the configuration options and syntax, see :ref:`configuration options `. diff --git a/docs/development/contributing.rst b/docs/development/contributing.rst index b0ad94d0a..695dde7be 100644 --- a/docs/development/contributing.rst +++ b/docs/development/contributing.rst @@ -82,8 +82,6 @@ a translation. To add them, do run ``makemessages`` from the ``data/static/`` di .. code-block:: bash cd data/static/ - # if django complains about not finding VAPID files when running from this bizare directory, - # prepend VAPID_PRIVATE_KEY_PATH="" to the following command python ../../manage.py makemessages --all -d djangojs --ignore jsi18n --ignore admin --ignore CACHE --ignore recurrence --ignore select2 We tend to edit our .po files using weblate, but a local editor like poedit works as well. \ No newline at end of file diff --git a/ephios/settings.py b/ephios/settings.py index 5bafad409..b6dd58ba8 100644 --- a/ephios/settings.py +++ b/ephios/settings.py @@ -30,12 +30,8 @@ environ.Env.read_env(env_file=env_path) DEBUG = env.bool("DEBUG") -if DEBUG: - DATA_DIR = env.str("DATA_DIR", os.path.join(BASE_DIR, "data")) -else: - # DATA_DIR must be set explicitly in production - DATA_DIR = env.str("DATA_DIR") +DATA_DIR = env.str("DATA_DIR") LOG_DIR = env.str("LOG_DIR", os.path.join(DATA_DIR, "logs")) MEDIA_ROOT = env.str("MEDIA_ROOT", os.path.join(DATA_DIR, "media")) STATIC_ROOT = env.str("STATIC_ROOT", os.path.join(DATA_DIR, "static")) From 7a13d499b930b25179b5c4e3714b61c3f964acc5 Mon Sep 17 00:00:00 2001 From: Felix Rindt Date: Wed, 13 Sep 2023 10:30:57 +0200 Subject: [PATCH 06/14] add healthcheck --- .env.example | 3 +-- Dockerfile | 3 ++- ephios/core/urls.py | 8 +++++--- ephios/core/views/healthcheck.py | 21 +++++++++++++++++++++ ephios/settings.py | 2 +- tests/core/test_views_healthcheck.py | 2 ++ 6 files changed, 32 insertions(+), 7 deletions(-) create mode 100644 ephios/core/views/healthcheck.py create mode 100644 tests/core/test_views_healthcheck.py diff --git a/.env.example b/.env.example index 40512c5ff..a1777254d 100644 --- a/.env.example +++ b/.env.example @@ -1,9 +1,8 @@ DEBUG=True -DATA_DIR=data DATABASE_URL=sqlite:///data/db.sqlite3 ALLOWED_HOSTS="*" SITE_URL=http://localhost:8000 -EMAIL_URL=dummymail:// +EMAIL_URL=consolemail:// DEFAULT_FROM_EMAIL=webmaster@localhost SERVER_EMAIL=root@localhost ADMINS="Root User " diff --git a/Dockerfile b/Dockerfile index d45cad8f6..279337fc3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,4 +41,5 @@ COPY deployment/docker/cron.sh /usr/local/bin/cron.sh RUN chmod +x /usr/local/bin/cron.sh ENTRYPOINT ["ephios"] -CMD ["run"] \ No newline at end of file +CMD ["run"] +HEALTHCHECK CMD curl -f http://localhost:80/healthcheck || exit 1 \ No newline at end of file diff --git a/ephios/core/urls.py b/ephios/core/urls.py index 27ebc5d0d..8654a3b66 100644 --- a/ephios/core/urls.py +++ b/ephios/core/urls.py @@ -39,6 +39,7 @@ EventTypeListView, EventTypeUpdateView, ) +from ephios.core.views.healthcheck import HealthcheckView from ephios.core.views.log import LogView from ephios.core.views.pwa import OfflineView, PWAManifestView, ServiceWorkerView from ephios.core.views.settings import ( @@ -68,6 +69,10 @@ app_name = "core" urlpatterns = [ path("", HomeView.as_view(), name="home"), + path("manifest.json", PWAManifestView.as_view(), name="pwa_manifest"), + path("serviceworker.js", ServiceWorkerView.as_view(), name="pwa_serviceworker"), + path("offline/", OfflineView.as_view(), name="pwa_offline"), + path("healthcheck/", HealthcheckView.as_view(), name="healthcheck"), path("events/", EventListView.as_view(), name="event_list"), path( "events//edit/", @@ -225,9 +230,6 @@ name="consequence_edit", ), path("log/", LogView.as_view(), name="log"), - path("manifest.json", PWAManifestView.as_view(), name="pwa_manifest"), - path("serviceworker.js", ServiceWorkerView.as_view(), name="pwa_serviceworker"), - path("offline/", OfflineView.as_view(), name="pwa_offline"), path("workinghours/own/", OwnWorkingHourView.as_view(), name="workinghours_own"), path( "workinghours/own/request/", diff --git a/ephios/core/views/healthcheck.py b/ephios/core/views/healthcheck.py new file mode 100644 index 000000000..9f4ffeee9 --- /dev/null +++ b/ephios/core/views/healthcheck.py @@ -0,0 +1,21 @@ +from django.contrib.auth.models import Permission +from django.core import cache +from django.db import Error as DjangoDBError +from django.http import HttpResponse +from django.views import View + + +class HealthcheckView(View): + def get(self, request, *args, **kwargs): + # check db access + try: + Permission.objects.exists() + except DjangoDBError: + return HttpResponse("DB not available.", status=503) + + # check cache access + cache.cache.set("_healthcheck", "1") + if not cache.cache.get("_healthcheck") == "1": + return HttpResponse("Cache not available.", status=503) + + return HttpResponse("OK", status=200) diff --git a/ephios/settings.py b/ephios/settings.py index b6dd58ba8..893e3f634 100644 --- a/ephios/settings.py +++ b/ephios/settings.py @@ -31,7 +31,7 @@ DEBUG = env.bool("DEBUG") -DATA_DIR = env.str("DATA_DIR") +DATA_DIR = env.str("DATA_DIR", os.path.join(BASE_DIR, "data")) LOG_DIR = env.str("LOG_DIR", os.path.join(DATA_DIR, "logs")) MEDIA_ROOT = env.str("MEDIA_ROOT", os.path.join(DATA_DIR, "media")) STATIC_ROOT = env.str("STATIC_ROOT", os.path.join(DATA_DIR, "static")) diff --git a/tests/core/test_views_healthcheck.py b/tests/core/test_views_healthcheck.py new file mode 100644 index 000000000..e0c1c0d1b --- /dev/null +++ b/tests/core/test_views_healthcheck.py @@ -0,0 +1,2 @@ +def test_healthcheck(django_app): + django_app.get("/healthcheck/", status=200) From ed75d5876a5020d1513f427eb5c3b506e9913488 Mon Sep 17 00:00:00 2001 From: Felix Rindt Date: Wed, 13 Sep 2023 11:45:59 +0200 Subject: [PATCH 07/14] add directory location check --- deployment/compose/nginx.conf | 2 +- docs/admin/configuration/index.rst | 146 +++++++++++++++++-------- docs/admin/deployment/docker/index.rst | 3 +- docs/admin/deployment/index.rst | 4 + docs/admin/deployment/manual/index.rst | 10 +- docs/development/contributing.rst | 6 +- ephios/core/apps.py | 1 + ephios/core/checks.py | 26 +++++ ephios/core/views/healthcheck.py | 5 +- ephios/extra/secrets.py | 20 ++++ ephios/settings.py | 44 ++++---- 11 files changed, 191 insertions(+), 76 deletions(-) create mode 100644 ephios/core/checks.py create mode 100644 ephios/extra/secrets.py diff --git a/deployment/compose/nginx.conf b/deployment/compose/nginx.conf index 90a575267..304922765 100644 --- a/deployment/compose/nginx.conf +++ b/deployment/compose/nginx.conf @@ -11,7 +11,7 @@ server { } location /static/ { - alias /var/ephios/data/static/; + alias /var/ephios/data/public/static/; access_log off; expires 1d; add_header Cache-Control "public"; diff --git a/docs/admin/configuration/index.rst b/docs/admin/configuration/index.rst index 3997714fa..c54260fae 100644 --- a/docs/admin/configuration/index.rst +++ b/docs/admin/configuration/index.rst @@ -10,71 +10,110 @@ for a more in-depth explanation of what they do. .. _env_file_options: -The following variables are available (plugins and some niche features might require additional environment variables): +The following variables are available (plugins and some niche features might require additional environment variables). + +Setup +----- `ENV_PATH`: Path to an environment file. Defaults to `.env` in the location of the ephios package. We recommend setting most of the following variables in this file. -`SECRET_KEY`: - **Required**. Django secret key used to encrypt session data and other sensitive information. +`DJANGO_SETTINGS_MODULE`: + Defaults to `ephios.settings`. If you want to use your own settings file, + set this to the path to your settings file. This variable cannot be set in the environment file. + +Debugging +--------- `DEBUG`: **Required**. Set to `True` to enable debug mode. Must be `False` in production. -`ALLOWED_HOSTS`: - **Required**. Comma-separated list of hostnames that are allowed to access the ephios instance. +`DEBUG_TOOLBAR`: + Set to `True` to enable the django debug toolbar. Must be `False` in production. + Defaults to `False`. -`DATABASE_URL`: - **Required**. URL to the database. See - `django-environ `__ for details. +`INTERNAL_IPS`: + Comma-separated list of IP addresses that are allowed to access the debug toolbar. + Defaults to `127.0.0.1`. -`SITE_URL`: - **Required**. URL used to construct absolute URLs in emails and other places. +Secrets +------- -`CACHE_URL`: - URL to the cache. See - `django-environ `__ for details. +`SECRET_KEY`: + **Important**. Django secret key used to encrypt session data and other sensitive information. + Defaults to a random value persisted into `PRIVATE_DIR/.secret`. -`STATIC_ROOT`: - **Required**: Path where static files are collected to. - A reverse proxy should be configured to serve them at `STATIC_URL`. +`VAPID_PRIVATE_KEY_PATH`: + Path to the private key used to sign web push notifications. If not provided, web push notifications wont work + on some platforms. See :ref:`web_push_notifications` for details. + Defaults to `PRIVATE_DIR/vapid_key.pem`. A keypair is automatically generated if it does not exist. -`EMAIL_URL`: - **Required**. URL to the email smtp server. See - `django-environ `__ for details. +Data storage and Logging +------------------------ -`DEFAULT_FROM_EMAIL`: - **Required**. Email address that is used as the sender for all - emails sent by ephios. (`Django docs `__) +`DATA_DIR`: + **Important**. Base path where ephios defaults to store files. + Defaults to a `data` folder in the location of the ephios package, + which is not recommended for production use. -`SERVER_EMAIL`: - **Required**. Email address that is used as the sender for all - error emails sent by django. (`Django docs `__) +`PUBLIC_DIR`: + Path where public files are stored. Defaults to `DATA_DIR/public`. -`ADMINS`: - **Required**. Email addresses that receive error emails. +`STATIC_ROOT`: + Path where static files (css/js) are collected to. + A reverse proxy should be configured to serve them at `STATIC_URL`. + Defaults to `PUBLIC_DIR/static`. + +`PRIVATE_DIR`: + Path where private files are stored. Defaults to `DATA_DIR/private`. + Make sure access to this folder is restricted to the user running ephios. + +`MEDIA_ROOT`: + Path where uploaded files are stored. + Defaults to `PRIVATE_DIR/media`. + You should backup this folder regularly. -`LOGGING_FILE`: - Path to the log file. If provided, ephios logs to this file. - The file is rotated daily. +`LOG_DIR`: + Path to the folder where log files are put. Files inside are rotated daily. + Defaults to `PRIVATE_DIR/logs`. `LOGGING_BACKUP_DAYS`: Number of days to keep log files. Defaults to 14. + +Database and Caching +-------------------- + +`DATABASE_URL`: + **Required**. URL to the database. See + `django-environ `__ for details. + +`CONN_MAX_AGE`: + Number of seconds to keep database connections open. Defaults to 0, meaning connections are closed after each request. + Refer to the `django docs `__ for details. + +`CACHE_URL`: + URL to the cache. We recommend redis. See + `django-environ `__ for details. + +URLs and Routing +---------------- + +`ALLOWED_HOSTS`: + **Required**. Comma-separated list of hostnames that are allowed to access the ephios instance. + + +`SITE_URL`: + **Required**. URL used to construct absolute URLs in emails and other places. + `STATIC_URL`: URL where the static files can be found by browsers. Defaults to ``/static/``, meaning they are served by the same host. -`DEBUG_TOOLBAR`: - Set to `True` to enable the django debug toolbar. Must be `False` in production. +Security +-------- -`INTERNAL_IPS`: - Comma-separated list of IP addresses that are allowed to access the debug toolbar. - -`VAPID_PRIVATE_KEY_PATH`: - Path to the private key used to sign web push notifications. If not provided, web push notifications wont work - on some platforms. See :ref:`web_push_notifications` for details. `SECURE_HSTS_SECONDS`: Number of seconds to set the `Strict-Transport-Security `__ @@ -88,10 +127,29 @@ The following variables are available (plugins and some niche features might req Set the `preload `__ flag in the `Strict-Transport-Security `__ header. Defaults to `False`. -`CONN_MAX_AGE`: - Number of seconds to keep database connections open. Defaults to 0, meaning connections are closed after each request. - Refer to the `django docs `__ for details. +`TRUST_X_FORWARDED_PROTO`: + ephios must be served over HTTPS in production. In some setups, ephios is behind a reverse proxy that terminates + SSL connections and the Origin header is not set with a https scheme. In this case, the proxy can communicate + the fact that the connection is secure by setting the + `X-Forwarded-Proto `__ header. + Then this setting must be set to `True`. See + `django docs `__ + for details. Defaults to `False`. -`DJANGO_SETTINGS_MODULE`: - Defaults to `ephios.settings`. If you want to use your own settings file, - set this to the path to your settings file. This variable cannot be set in the environment file. +E-Mail +------ + +`EMAIL_URL`: + **Required**. URL to the email smtp server. See + `django-environ `__ for details. + +`DEFAULT_FROM_EMAIL`: + **Required**. Email address that is used as the sender for all + emails sent by ephios. (`Django docs `__) + +`SERVER_EMAIL`: + **Required**. Email address that is used as the sender for all + error emails sent by django. (`Django docs `__) + +`ADMINS`: + **Required**. Email addresses that receive error emails. diff --git a/docs/admin/deployment/docker/index.rst b/docs/admin/deployment/docker/index.rst index 1d588ffb3..de32fe1dd 100644 --- a/docs/admin/deployment/docker/index.rst +++ b/docs/admin/deployment/docker/index.rst @@ -7,7 +7,8 @@ Docker Image We automatically build a Docker image for every release. It uses gunicorn as a WSGI-server and still requires a database, redis and some ssl-terminating webserver/proxy -to be set up. +to be set up. Also, static files need to be served by +a webserver from a common volume. The image is available on the Github Container Registry. diff --git a/docs/admin/deployment/index.rst b/docs/admin/deployment/index.rst index e293c4131..4938dc91e 100644 --- a/docs/admin/deployment/index.rst +++ b/docs/admin/deployment/index.rst @@ -67,6 +67,10 @@ Restart and maybe also add your domain to the `HSTS preload list + Alias /static/ /var/ephios/data/public/static/ + Require all granted diff --git a/docs/development/contributing.rst b/docs/development/contributing.rst index 695dde7be..ae1069ef1 100644 --- a/docs/development/contributing.rst +++ b/docs/development/contributing.rst @@ -76,12 +76,12 @@ the `locale/**/.po` files for translation: Calling ``makemessages`` in the ``djangojs`` domain will find gettext calls in javascript files in the current working directory. Therefore, we need to ignore the ``data`` which contains static files from 3rd-party-packages already translated and the ``docs`` directory. Some 3rd-party-javascript comes without -a translation. To add them, do run ``makemessages`` from the ``data/static/`` directory after running +a translation. To add them, do run ``makemessages`` from the ``data/public/static/`` directory after running ``collectstatic``, but ignore all directories of 3rd-party-packages that are already translated, e.g.: .. code-block:: bash - cd data/static/ - python ../../manage.py makemessages --all -d djangojs --ignore jsi18n --ignore admin --ignore CACHE --ignore recurrence --ignore select2 + cd data/public/static/ + python ../../../manage.py makemessages --all -d djangojs --ignore jsi18n --ignore admin --ignore CACHE --ignore recurrence --ignore select2 We tend to edit our .po files using weblate, but a local editor like poedit works as well. \ No newline at end of file diff --git a/ephios/core/apps.py b/ephios/core/apps.py index 49d1ffd7b..9db8a1259 100644 --- a/ephios/core/apps.py +++ b/ephios/core/apps.py @@ -13,6 +13,7 @@ class CoreAppConfig(AppConfig): def ready(self): from ephios.core.dynamic_preferences_registry import event_type_preference_registry + from . import checks # pylint: disable=unused-import from . import signals # pylint: disable=unused-import EventTypePreference = self.get_model("EventTypePreference") diff --git a/ephios/core/checks.py b/ephios/core/checks.py new file mode 100644 index 000000000..d7b11c376 --- /dev/null +++ b/ephios/core/checks.py @@ -0,0 +1,26 @@ +import os + +from django.conf import settings +from django.core.checks import Warning, register + + +@register("ephios", deploy=True) +def check_data_dir_is_not_inside_base_dir(app_configs, **kwargs): + """ + On production setups, the data directory should + not be inside the base directory (ephios package directory). + """ + errors = [] + for name, directory in settings.DIRECTORIES.items(): + if os.path.commonpath([settings.BASE_DIR, directory]) == settings.BASE_DIR: + errors.append( + Warning( + "ephios data is stored near the ephios package directory.", + hint=f"You probably don't want to have {name} at {directory} to " + f"be inside or next to the ephios python package at " + f"{settings.BASE_DIR}! " + f"Configure ephios to use another location.", + id="ephios.W001", + ) + ) + return errors diff --git a/ephios/core/views/healthcheck.py b/ephios/core/views/healthcheck.py index 9f4ffeee9..471229312 100644 --- a/ephios/core/views/healthcheck.py +++ b/ephios/core/views/healthcheck.py @@ -7,9 +7,11 @@ class HealthcheckView(View): def get(self, request, *args, **kwargs): + messages = [] # check db access try: Permission.objects.exists() + messages.append("DB OK") except DjangoDBError: return HttpResponse("DB not available.", status=503) @@ -17,5 +19,6 @@ def get(self, request, *args, **kwargs): cache.cache.set("_healthcheck", "1") if not cache.cache.get("_healthcheck") == "1": return HttpResponse("Cache not available.", status=503) + messages.append("Cache OK") - return HttpResponse("OK", status=200) + return HttpResponse("
".join(messages), status=200) diff --git a/ephios/extra/secrets.py b/ephios/extra/secrets.py new file mode 100644 index 000000000..ff1dda0a0 --- /dev/null +++ b/ephios/extra/secrets.py @@ -0,0 +1,20 @@ +import os +import string + +from django.utils.crypto import get_random_string + + +def django_secret_from_file(path: str): + if os.path.exists(path): + with open(path, "r") as f: + return f.read().strip() + chars = string.ascii_letters + string.digits + string.punctuation + secret = get_random_string(50, chars) + with open(path, "w") as f: + os.chmod(path, 0o600) + try: + os.chown(path, os.getuid(), os.getgid()) + except AttributeError: + pass # os.chown is not available on Windows + f.write(secret) + return secret diff --git a/ephios/settings.py b/ephios/settings.py index 893e3f634..4e69a5698 100644 --- a/ephios/settings.py +++ b/ephios/settings.py @@ -1,7 +1,6 @@ import copy import datetime import os -import string from datetime import timedelta from email.utils import getaddresses from pathlib import Path @@ -9,10 +8,11 @@ import environ from cryptography.hazmat.primitives import serialization from django.contrib.messages import constants -from django.utils.crypto import get_random_string from django.utils.translation import gettext_lazy from py_vapid import Vapid, b64urlencode +from ephios.extra.secrets import django_secret_from_file + try: import importlib_metadata # importlib is broken on python3.8, using backport except ImportError: @@ -32,31 +32,31 @@ DEBUG = env.bool("DEBUG") DATA_DIR = env.str("DATA_DIR", os.path.join(BASE_DIR, "data")) -LOG_DIR = env.str("LOG_DIR", os.path.join(DATA_DIR, "logs")) -MEDIA_ROOT = env.str("MEDIA_ROOT", os.path.join(DATA_DIR, "media")) -STATIC_ROOT = env.str("STATIC_ROOT", os.path.join(DATA_DIR, "static")) +PUBLIC_DIR = env.str("PUBLIC_DIR", os.path.join(DATA_DIR, "public")) +STATIC_ROOT = env.str("STATIC_ROOT", os.path.join(PUBLIC_DIR, "static")) + +PRIVATE_DIR = env.str("PRIVATE_DIR", os.path.join(DATA_DIR, "private")) +LOG_DIR = env.str("LOG_DIR", os.path.join(PRIVATE_DIR, "logs")) +MEDIA_ROOT = env.str("MEDIA_ROOT", os.path.join(PRIVATE_DIR, "media")) + +DIRECTORIES = { + "DATA_DIR": DATA_DIR, + "PUBLIC_DIR": PUBLIC_DIR, + "STATIC_ROOT": STATIC_ROOT, + "PRIVATE_DIR": PRIVATE_DIR, + "LOG_DIR": LOG_DIR, + "MEDIA_ROOT": MEDIA_ROOT, +} -for path in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT]: +for path in DIRECTORIES.values(): if not os.path.exists(path): os.makedirs(path, exist_ok=True) if "SECRET_KEY" in env: SECRET_KEY = env.str("SECRET_KEY") else: - SECRET_FILE = os.path.join(DATA_DIR, ".secret") - if os.path.exists(SECRET_FILE): - with open(SECRET_FILE, "r") as f: - SECRET_KEY = f.read().strip() - else: - chars = string.ascii_letters + string.digits + string.punctuation - SECRET_KEY = get_random_string(50, chars) - with open(SECRET_FILE, "w") as f: - os.chmod(SECRET_FILE, 0o600) - try: - os.chown(SECRET_FILE, os.getuid(), os.getgid()) - except AttributeError: - pass # os.chown is not available on Windows - f.write(SECRET_KEY) + SECRET_FILE = os.path.join(PRIVATE_DIR, ".secret") + SECRET_KEY = django_secret_from_file(SECRET_FILE) ALLOWED_HOSTS = env.list("ALLOWED_HOSTS") @@ -382,7 +382,9 @@ def GET_SITE_URL(): ] # django-webpush -VAPID_PRIVATE_KEY_PATH = env.str("VAPID_PRIVATE_KEY_PATH", os.path.join(DATA_DIR, "vapid_key.pem")) +VAPID_PRIVATE_KEY_PATH = env.str( + "VAPID_PRIVATE_KEY_PATH", os.path.join(PRIVATE_DIR, "vapid_key.pem") +) if not os.path.exists(VAPID_PRIVATE_KEY_PATH): vapid = Vapid() vapid.generate_keys() From bb21347342ac34ede6141b9622ca35d446560760 Mon Sep 17 00:00:00 2001 From: Felix Rindt Date: Wed, 13 Sep 2023 14:02:28 +0200 Subject: [PATCH 08/14] add healthcheck --- Dockerfile | 7 +-- docs/admin/deployment/manual/index.rst | 2 +- ephios/core/dynamic_preferences_registry.py | 39 ++++++++++++++ ephios/core/signals.py | 8 +++ .../core/settings/settings_instance.html | 51 ++++++++++++++++++- ephios/core/views/healthcheck.py | 24 +++++++-- ephios/core/views/settings.py | 13 +++++ ephios/settings.py | 15 +++--- poetry.lock | 2 +- pyproject.toml | 2 +- 10 files changed, 144 insertions(+), 19 deletions(-) diff --git a/Dockerfile b/Dockerfile index 279337fc3..a41842a33 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,11 +27,8 @@ RUN mkdir -p /var/ephios/data/ && \ mkdir -p /var/log/supervisord/ && \ mkdir -p /var/run/supervisord/ -#COPY pyproject.toml poetry.lock .git /usr/src/ephios/ -#RUN poetry install -E pgsql -E redis -E mysql -# good caching point COPY . /usr/src/ephios -RUN poetry install -E pgsql -E redis -E mysql +RUN poetry install --all-extras --without=dev COPY deployment/docker/entrypoint.sh /usr/local/bin/ephios RUN chmod +x /usr/local/bin/ephios @@ -42,4 +39,4 @@ RUN chmod +x /usr/local/bin/cron.sh ENTRYPOINT ["ephios"] CMD ["run"] -HEALTHCHECK CMD curl -f http://localhost:80/healthcheck || exit 1 \ No newline at end of file +HEALTHCHECK CMD curl -f http://localhost:80/healthcheck || exit 1 diff --git a/docs/admin/deployment/manual/index.rst b/docs/admin/deployment/manual/index.rst index 69f4191d0..db794f7e2 100644 --- a/docs/admin/deployment/manual/index.rst +++ b/docs/admin/deployment/manual/index.rst @@ -118,7 +118,7 @@ Now that the configuration is in place, we can build the static files and the tr Setup cron '''''''''' -ephios needs to have the ``run_periodic`` management command run periodically (every few minutes). +ephios needs to have the ``run_periodic`` management command run periodically (at least every five minutes). This command sends notifications and performs other tasks that need to be done regularly. Run ``crontab -e -u ephios`` and add the following line: diff --git a/ephios/core/dynamic_preferences_registry.py b/ephios/core/dynamic_preferences_registry.py index c9a661faf..a8c5c53ef 100644 --- a/ephios/core/dynamic_preferences_registry.py +++ b/ephios/core/dynamic_preferences_registry.py @@ -1,5 +1,10 @@ +from datetime import datetime, timedelta + +from django.conf import settings from django.contrib.auth.models import Group +from django.utils import timezone from django.utils.safestring import mark_safe +from django.utils.timezone import make_aware from django.utils.translation import gettext_lazy as _ from django_select2.forms import Select2MultipleWidget from dynamic_preferences.preferences import Section @@ -8,6 +13,7 @@ global_preferences_registry, ) from dynamic_preferences.types import ( + DateTimePreference, ModelMultipleChoicePreference, MultipleChoicePreference, StringPreference, @@ -31,6 +37,7 @@ class EventTypeRegistry(PerInstancePreferenceRegistry): notifications_user_section = Section("notifications") responsible_notifications_user_section = Section("responsible_notifications") general_global_section = Section("general") +internal_section = Section("internal") # for settings/stats that should not be exposed to users @global_preferences_registry.register @@ -62,6 +69,38 @@ def get_choices(): ] +@global_preferences_registry.register +class LastRunPeriodicCall(DateTimePreference): + NONE_VALUE = make_aware(datetime(1970, 1, 1)) + name = "last_run_periodic_call" + verbose_name = _("Last run periodic call") + section = internal_section + required = False + + def get_default(self): + return self.NONE_VALUE + + @classmethod + def get_last_call(cls): + preferences = global_preferences_registry.manager() + last_call = preferences[f"{cls.section.name}__{cls.name}"] + if last_call == cls.NONE_VALUE: + return None + return last_call + + @classmethod + def is_stuck(cls): + last_call = cls.get_last_call() + return last_call is None or ( + (timezone.now() - last_call) > timedelta(seconds=settings.RUN_PERIODIC_MAX_INTERVAL) + ) + + @classmethod + def set_last_call(cls, value): + preferences = global_preferences_registry.manager() + preferences[f"{cls.section.name}__{cls.name}"] = value + + @user_preferences_registry.register class NotificationPreference(JSONPreference): name = "notifications" diff --git a/ephios/core/signals.py b/ephios/core/signals.py index bfe333cbd..366627e5a 100644 --- a/ephios/core/signals.py +++ b/ephios/core/signals.py @@ -1,4 +1,5 @@ from django.dispatch import receiver +from django.utils import timezone from ephios.core.plugins import PluginSignal from ephios.core.services.notifications.backends import send_all_notifications @@ -174,6 +175,13 @@ def send_notifications(sender, **kwargs): send_all_notifications() +@receiver(periodic_signal, dispatch_uid="ephios.core.signals.update_last_run_periodic_call") +def update_last_run_periodic_call(sender, **kwargs): + from ephios.core.dynamic_preferences_registry import LastRunPeriodicCall + + LastRunPeriodicCall.set_last_call(timezone.now()) + + periodic_signal.connect( send_participation_finished, dispatch_uid="ephios.core.signals.send_participation_finished" ) diff --git a/ephios/core/templates/core/settings/settings_instance.html b/ephios/core/templates/core/settings/settings_instance.html index aa540c464..391f8cfe3 100644 --- a/ephios/core/templates/core/settings/settings_instance.html +++ b/ephios/core/templates/core/settings/settings_instance.html @@ -1,11 +1,60 @@ {% extends "core/settings/settings_base.html" %} +{% load humanize %} {% load crispy_forms_filters %} {% load i18n %} {% block settings_content %} -
+ {% csrf_token %} {{ form|crispy }}
+ + {% if show_system_health %} +
+

+ {% translate "System health" %} +

+
+
+
+ {% translate "Cron job" %} + {% if last_run_periodic_call_stuck %} + + {% else %} + + {% endif %} +
+

+ {% blocktranslate trimmed %} + A cron job must regularly call ephios to do recurring tasks + like sending reminder emails. + {% endblocktranslate %} + + + + +
+ {% if last_run_periodic_call == None %} + {% translate "Last run:" %} + + {% translate "never" %} + + {% elif last_run_periodic_call_stuck %} + {% translate "Last run:" %} + + {{ last_run_periodic_call|naturaltime }} + + {% else %} + {% translate "Last run:" %} {{ last_run_periodic_call|naturaltime }} + {% endif %} +

+
+
+
+ {% endif %} + {% endblock %} diff --git a/ephios/core/views/healthcheck.py b/ephios/core/views/healthcheck.py index 471229312..3b8b7e9c5 100644 --- a/ephios/core/views/healthcheck.py +++ b/ephios/core/views/healthcheck.py @@ -2,23 +2,41 @@ from django.core import cache from django.db import Error as DjangoDBError from django.http import HttpResponse +from django.utils.formats import date_format from django.views import View +from ephios.core.dynamic_preferences_registry import LastRunPeriodicCall + class HealthcheckView(View): def get(self, request, *args, **kwargs): messages = [] + errors = [] # check db access try: Permission.objects.exists() messages.append("DB OK") except DjangoDBError: - return HttpResponse("DB not available.", status=503) + errors.append("DB not available") # check cache access cache.cache.set("_healthcheck", "1") if not cache.cache.get("_healthcheck") == "1": - return HttpResponse("Cache not available.", status=503) - messages.append("Cache OK") + errors.append("Cache not available") + else: + messages.append("Cache OK") + + # check cronjob + if LastRunPeriodicCall.is_stuck(): + errors.append( + f"Cronjob stuck, last run {date_format(LastRunPeriodicCall.get_last_call(),format='SHORT_DATETIME_FORMAT')}" + ) + else: + messages.append("Cronjob OK") + + if errors: + return HttpResponse( + "
".join(errors) + "

" + "
".join(messages), status=503 + ) return HttpResponse("
".join(messages), status=200) diff --git a/ephios/core/views/settings.py b/ephios/core/views/settings.py index b9aac4811..9e6e19b17 100644 --- a/ephios/core/views/settings.py +++ b/ephios/core/views/settings.py @@ -6,6 +6,7 @@ from django.views.generic import FormView, TemplateView from dynamic_preferences.forms import global_preference_form_builder +from ephios.core.dynamic_preferences_registry import LastRunPeriodicCall from ephios.core.forms.users import UserNotificationPreferenceForm from ephios.core.signals import management_settings_sections from ephios.extra.mixins import StaffRequiredMixin @@ -55,6 +56,18 @@ def form_valid(self, form): def get_success_url(self): return reverse("core:settings_instance") + def get_context_data(self, **kwargs): + if self.request.user.is_superuser: + kwargs.update(self._get_healthcheck_context()) + return super().get_context_data(**kwargs) + + def _get_healthcheck_context(self): + return { + "show_system_health": True, + "last_run_periodic_call": LastRunPeriodicCall.get_last_call(), + "last_run_periodic_call_stuck": LastRunPeriodicCall.is_stuck(), + } + class PersonalDataSettingsView(LoginRequiredMixin, TemplateView): template_name = "core/settings/settings_personal_data.html" diff --git a/ephios/settings.py b/ephios/settings.py index 4e69a5698..a3af39dcd 100644 --- a/ephios/settings.py +++ b/ephios/settings.py @@ -402,9 +402,13 @@ def GET_SITE_URL(): "VAPID_ADMIN_EMAIL": ADMINS[0][1], } +# Health check +# interval for calls to the run_periodic_tasks management command over which the cronjob is considered to be broken +RUN_PERIODIC_MAX_INTERVAL = 60 * 5 + 30 # 5 minutes + 30 seconds -DEFAULT_LISTVIEW_PAGINATION = 100 +# django-rest-framework +DEFAULT_LISTVIEW_PAGINATION = 100 REST_FRAMEWORK = { "DEFAULT_PERMISSION_CLASSES": ["rest_framework.permissions.DjangoObjectPermissions"], "DEFAULT_VERSIONING_CLASS": "rest_framework.versioning.NamespaceVersioning", @@ -440,9 +444,7 @@ def GET_SITE_URL(): "REFRESH_TOKEN_EXPIRE_SECONDS": timedelta(days=90), } -""" -SECURITY SETTINGS -""" +# SECURITY SETTINGS if not DEBUG: SESSION_COOKIE_SECURE = True CSRF_COOKIE_SECURE = True @@ -459,9 +461,8 @@ def GET_SITE_URL(): if env.bool("TRUST_X_FORWARDED_PROTO", default=False): SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") -""" -OIDC SETTINGS -""" + +# OIDC SETTINGS if ENABLE_OIDC_CLIENT := env.bool("ENABLE_OIDC_CLIENT", False): INSTALLED_APPS.append("mozilla_django_oidc") AUTHENTICATION_BACKENDS.append("ephios.extra.auth.EphiosOIDCAB") diff --git a/poetry.lock b/poetry.lock index ddafa7400..2f2dc73f2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2694,4 +2694,4 @@ redis = ["redis"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "e7c2bba0b7fb7204afb028fa569e759f95b52d0f8edde6e736dede15d82baa8e" +content-hash = "6254c638d6bcd04a3b1501c7a5bd7d68cacd98dd762fd775434c192464805bef" diff --git a/pyproject.toml b/pyproject.toml index 282bb8635..aacd42f2e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ mysqlclient = {version = "^2.1.1", optional = true} redis = {extras = ["hiredis"], version = "^5.0.0", optional = true} django-compressor = "^4.3" django-statici18n = "^2.3.1" -django-dynamic-preferences = "^1.13.0" +django-dynamic-preferences = "^1.15.0" django-crispy-forms = "^2.0" django-webpush = "^0.3.5" importlib-metadata = { version = ">=5.2.0", python = "<3.9" } From e436bd443c0a1e4bdcd4335ad6cd6e460d092f22 Mon Sep 17 00:00:00 2001 From: Felix Rindt Date: Wed, 13 Sep 2023 14:59:38 +0200 Subject: [PATCH 09/14] linting --- .github/workflows/publish.yml | 4 ++-- ephios/core/checks.py | 5 +++-- ephios/core/views/healthcheck.py | 9 ++++++--- ephios/extra/secrets.py | 4 ++-- tests/core/test_views_healthcheck.py | 2 -- 5 files changed, 13 insertions(+), 11 deletions(-) delete mode 100644 tests/core/test_views_healthcheck.py diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 990fd7dd1..59f450a87 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,10 +1,10 @@ -name: Publish to pypi +name: Publish to PyPI on: push: tags: - 'v*.*.*' jobs: - build: + pypi: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/ephios/core/checks.py b/ephios/core/checks.py index d7b11c376..f6050bf61 100644 --- a/ephios/core/checks.py +++ b/ephios/core/checks.py @@ -1,7 +1,8 @@ import os from django.conf import settings -from django.core.checks import Warning, register +from django.core.checks import Warning as DjangoWarning +from django.core.checks import register @register("ephios", deploy=True) @@ -14,7 +15,7 @@ def check_data_dir_is_not_inside_base_dir(app_configs, **kwargs): for name, directory in settings.DIRECTORIES.items(): if os.path.commonpath([settings.BASE_DIR, directory]) == settings.BASE_DIR: errors.append( - Warning( + DjangoWarning( "ephios data is stored near the ephios package directory.", hint=f"You probably don't want to have {name} at {directory} to " f"be inside or next to the ephios python package at " diff --git a/ephios/core/views/healthcheck.py b/ephios/core/views/healthcheck.py index 3b8b7e9c5..4fdf15323 100644 --- a/ephios/core/views/healthcheck.py +++ b/ephios/core/views/healthcheck.py @@ -28,9 +28,12 @@ def get(self, request, *args, **kwargs): # check cronjob if LastRunPeriodicCall.is_stuck(): - errors.append( - f"Cronjob stuck, last run {date_format(LastRunPeriodicCall.get_last_call(),format='SHORT_DATETIME_FORMAT')}" - ) + if last_call := LastRunPeriodicCall.get_last_call(): + errors.append( + f"Cronjob stuck, last run {date_format(last_call,format='SHORT_DATETIME_FORMAT')}" + ) + else: + errors.append("Cronjob stuck, no last run") else: messages.append("Cronjob OK") diff --git a/ephios/extra/secrets.py b/ephios/extra/secrets.py index ff1dda0a0..9ca3a2cd5 100644 --- a/ephios/extra/secrets.py +++ b/ephios/extra/secrets.py @@ -6,11 +6,11 @@ def django_secret_from_file(path: str): if os.path.exists(path): - with open(path, "r") as f: + with open(path, "r", encoding="utf8") as f: return f.read().strip() chars = string.ascii_letters + string.digits + string.punctuation secret = get_random_string(50, chars) - with open(path, "w") as f: + with open(path, "w", encoding="utf8") as f: os.chmod(path, 0o600) try: os.chown(path, os.getuid(), os.getgid()) diff --git a/tests/core/test_views_healthcheck.py b/tests/core/test_views_healthcheck.py deleted file mode 100644 index e0c1c0d1b..000000000 --- a/tests/core/test_views_healthcheck.py +++ /dev/null @@ -1,2 +0,0 @@ -def test_healthcheck(django_app): - django_app.get("/healthcheck/", status=200) From b9a1b6bf9a3b72f7f3df6d0b2ae50c26a73ad0c9 Mon Sep 17 00:00:00 2001 From: Felix Rindt Date: Wed, 13 Sep 2023 14:59:45 +0200 Subject: [PATCH 10/14] docker workflow --- .github/workflows/docker.yml | 43 ++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 000000000..5d6116abd --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,43 @@ + +name: docker +on: + workflow_dispatch: + push: + branches: + - 'main' + tags: + - 'v*' + pull_request: + branches: + - 'main' + +jobs: + docker: + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/ephios-dev/ephios + + - name: Login to Github Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + push: true #${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} From 389a2e11fb3be11ecd234b951e4116e279b860be Mon Sep 17 00:00:00 2001 From: Felix Rindt Date: Wed, 13 Sep 2023 16:14:51 +0200 Subject: [PATCH 11/14] unify healthchecks --- ephios/core/services/health/healthchecks.py | 178 ++++++++++++++++++ ephios/core/signals.py | 6 + .../core/settings/settings_instance.html | 83 ++++---- ephios/core/urls.py | 4 +- ephios/core/views/healthcheck.py | 35 +--- ephios/core/views/settings.py | 11 +- tests/core/test_views_settings.py | 6 + 7 files changed, 241 insertions(+), 82 deletions(-) create mode 100644 ephios/core/services/health/healthchecks.py diff --git a/ephios/core/services/health/healthchecks.py b/ephios/core/services/health/healthchecks.py new file mode 100644 index 000000000..ea14c22e7 --- /dev/null +++ b/ephios/core/services/health/healthchecks.py @@ -0,0 +1,178 @@ +import os +from pathlib import Path + +from django.conf import settings +from django.contrib.auth.models import Permission +from django.contrib.humanize.templatetags.humanize import naturaltime +from django.dispatch import receiver +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +from ephios.core.dynamic_preferences_registry import LastRunPeriodicCall +from ephios.core.signals import register_healthchecks + +# health checks are meant to monitor the health of the application while it is running +# in contrast there are django checks which are meant to check the configuration of the application + + +def run_healthchecks(): + for _, healthchecks in register_healthchecks.send(None): + for HealthCheck in healthchecks: + check = HealthCheck() + status, message = check.check() + yield check, status, message + + +class HealthCheckStatus: + OK = "ok" + WARNING = "warning" + ERROR = "error" + + +class AbstractHealthCheck: + @property + def slug(self): + """ + Return a unique slug for this health check. + """ + raise NotImplementedError + + @property + def name(self): + """ + Return a short name of this health check. + """ + raise NotImplementedError + + @property + def description(self): + """ + Return a short description of this health check. + """ + raise NotImplementedError + + @property + def documentation_link(self): + """ + Return a link to the documentation of this health check. + """ + return None + + def check(self): + """ + Return a tuple of (status, message) where status is one of HealthCheckStatus + """ + raise NotImplementedError + + +class DBHealthCheck(AbstractHealthCheck): + slug = "db" + name = _("Database") + description = _("The database is the central storage for all data.") + documentation_link = "https://docs.djangoproject.com/en/stable/ref/databases/" + + def check(self): + from django.db import connection + + try: + connection.cursor() + Permission.objects.exists() + except Exception as e: + return HealthCheckStatus.ERROR, str(e) + + if settings.DATABASES["default"]["ENGINE"] == "django.db.backends.sqlite3": + return HealthCheckStatus.WARNING, _( + "Using SQLite, this is not recommended in production." + ) + + return HealthCheckStatus.OK, _("Database connection established.") + + +class CacheHealthCheck(AbstractHealthCheck): + slug = "cache" + name = _("Cache") + description = _("The cache is used to store temporary data.") + documentation_link = "https://docs.djangoproject.com/en/stable/topics/cache/" + + def check(self): + from django.core import cache + + try: + cache.cache.set("_healthcheck", "1") + if not cache.cache.get("_healthcheck") == "1": + raise Exception("Cache not available") + except Exception as e: + return HealthCheckStatus.ERROR, str(e) + + if ( + settings.CACHES.get("default", {}).get("BACKEND") + == "django.core.cache.backends.locmem.LocMemCache" + ): + return HealthCheckStatus.WARNING, _( + "Using LocMemCache, this is not recommended in production." + ) + + return HealthCheckStatus.OK, _("Cache connection established.") + + +class CronJobHealthCheck(AbstractHealthCheck): + slug = "cronjob" + name = _("Cronjob") + description = _( + "A cron job must regularly call ephios to do recurring tasks like sending notifications." + ) + documentation_link = ( + "https://docs.ephios.de/en/stable/admin/deployment/manual/index.html#setup-cron" + ) + + def check(self): + last_call = LastRunPeriodicCall.get_last_call() + if LastRunPeriodicCall.is_stuck(): + if last_call: + return ( + HealthCheckStatus.WARNING, + mark_safe( + _("Cronjob stuck, last run {last_call}.").format( + last_call=naturaltime(last_call), + ) + ), + ) + else: + return ( + HealthCheckStatus.ERROR, + mark_safe(_("Cronjob stuck, no last run.")), + ) + else: + return ( + HealthCheckStatus.OK, + mark_safe(_("Last run {last_call}.").format(last_call=naturaltime(last_call))), + ) + + +class WritableMediaRootHealthCheck(AbstractHealthCheck): + slug = "writable_media_root" + name = _("Writable Media Root") + description = _("The media root must be writable by the application server.") + documentation_link = ( + "https://docs.ephios.de/en/stable/admin/deployment/manual/index.html#data-directory" + ) + + def check(self): + media_root = Path(settings.MEDIA_ROOT) + if not os.access(media_root, os.W_OK): + return ( + HealthCheckStatus.ERROR, + mark_safe(_("Media root not writable by application server.")), + ) + return ( + HealthCheckStatus.OK, + mark_safe(_("Media root writable by application server.")), + ) + + +@receiver(register_healthchecks, dispatch_uid="ephios.core.healthchecks.register_core_healthchecks") +def register_core_healthchecks(sender, **kwargs): + yield DBHealthCheck + yield CacheHealthCheck + yield CronJobHealthCheck + yield WritableMediaRootHealthCheck diff --git a/ephios/core/signals.py b/ephios/core/signals.py index 366627e5a..55563482e 100644 --- a/ephios/core/signals.py +++ b/ephios/core/signals.py @@ -95,6 +95,12 @@ Receivers should return a list of subclasses of ``ephios.core.notifications.backends.AbstractNotificationBackend`` """ +register_healthchecks = PluginSignal() +""" +This signal is sent out to get all health checks that can be run to monitor the health of the application. +Receivers should return a list of subclasses of ``ephios.core.services.health.AbstractHealthCheck`` +""" + periodic_signal = PluginSignal() """ This signal is called periodically, at least every 15 minutes. diff --git a/ephios/core/templates/core/settings/settings_instance.html b/ephios/core/templates/core/settings/settings_instance.html index 391f8cfe3..6adfce167 100644 --- a/ephios/core/templates/core/settings/settings_instance.html +++ b/ephios/core/templates/core/settings/settings_instance.html @@ -10,50 +10,47 @@ - {% if show_system_health %} -
-

- {% translate "System health" %} -

-
-
-
- {% translate "Cron job" %} - {% if last_run_periodic_call_stuck %} - - {% else %} - - {% endif %} -
-

- {% blocktranslate trimmed %} - A cron job must regularly call ephios to do recurring tasks - like sending reminder emails. - {% endblocktranslate %} - - - - -
- {% if last_run_periodic_call == None %} - {% translate "Last run:" %} - - {% translate "never" %} - - {% elif last_run_periodic_call_stuck %} - {% translate "Last run:" %} - - {{ last_run_periodic_call|naturaltime }} - - {% else %} - {% translate "Last run:" %} {{ last_run_periodic_call|naturaltime }} - {% endif %} -

+ {% if healthchecks %} + +

+ {% translate "System health" %} +

+
+ {% for check, status, message in healthchecks %} +
+
+
+
+ {{ check.name }} + {% if status == "error" %} + + + {% elif status == "warning" %} + + + {% elif status == "ok" %} + + + {% endif %} +
+

+ {{ check.description }} + {% if check.documentation_link %} + + + + + {% endif %} +

+

+ {{ message }} +

+
+
-
+ {% endfor %}
{% endif %} diff --git a/ephios/core/urls.py b/ephios/core/urls.py index 8654a3b66..a1bbf8830 100644 --- a/ephios/core/urls.py +++ b/ephios/core/urls.py @@ -39,7 +39,7 @@ EventTypeListView, EventTypeUpdateView, ) -from ephios.core.views.healthcheck import HealthcheckView +from ephios.core.views.healthcheck import HealthCheckView from ephios.core.views.log import LogView from ephios.core.views.pwa import OfflineView, PWAManifestView, ServiceWorkerView from ephios.core.views.settings import ( @@ -72,7 +72,7 @@ path("manifest.json", PWAManifestView.as_view(), name="pwa_manifest"), path("serviceworker.js", ServiceWorkerView.as_view(), name="pwa_serviceworker"), path("offline/", OfflineView.as_view(), name="pwa_offline"), - path("healthcheck/", HealthcheckView.as_view(), name="healthcheck"), + path("healthcheck/", HealthCheckView.as_view(), name="healthcheck"), path("events/", EventListView.as_view(), name="event_list"), path( "events//edit/", diff --git a/ephios/core/views/healthcheck.py b/ephios/core/views/healthcheck.py index 4fdf15323..23310d8bb 100644 --- a/ephios/core/views/healthcheck.py +++ b/ephios/core/views/healthcheck.py @@ -1,41 +1,20 @@ -from django.contrib.auth.models import Permission -from django.core import cache -from django.db import Error as DjangoDBError from django.http import HttpResponse -from django.utils.formats import date_format from django.views import View -from ephios.core.dynamic_preferences_registry import LastRunPeriodicCall +from ephios.core.services.health.healthchecks import HealthCheckStatus, run_healthchecks -class HealthcheckView(View): +class HealthCheckView(View): def get(self, request, *args, **kwargs): messages = [] errors = [] - # check db access - try: - Permission.objects.exists() - messages.append("DB OK") - except DjangoDBError: - errors.append("DB not available") - # check cache access - cache.cache.set("_healthcheck", "1") - if not cache.cache.get("_healthcheck") == "1": - errors.append("Cache not available") - else: - messages.append("Cache OK") - - # check cronjob - if LastRunPeriodicCall.is_stuck(): - if last_call := LastRunPeriodicCall.get_last_call(): - errors.append( - f"Cronjob stuck, last run {date_format(last_call,format='SHORT_DATETIME_FORMAT')}" - ) + for check, status, message in run_healthchecks(): + text = f"{check.name}: {message}" + if status == HealthCheckStatus.OK: + messages.append(text) else: - errors.append("Cronjob stuck, no last run") - else: - messages.append("Cronjob OK") + errors.append(text) if errors: return HttpResponse( diff --git a/ephios/core/views/settings.py b/ephios/core/views/settings.py index 9e6e19b17..92bc336be 100644 --- a/ephios/core/views/settings.py +++ b/ephios/core/views/settings.py @@ -6,8 +6,8 @@ from django.views.generic import FormView, TemplateView from dynamic_preferences.forms import global_preference_form_builder -from ephios.core.dynamic_preferences_registry import LastRunPeriodicCall from ephios.core.forms.users import UserNotificationPreferenceForm +from ephios.core.services.health.healthchecks import run_healthchecks from ephios.core.signals import management_settings_sections from ephios.extra.mixins import StaffRequiredMixin @@ -58,16 +58,9 @@ def get_success_url(self): def get_context_data(self, **kwargs): if self.request.user.is_superuser: - kwargs.update(self._get_healthcheck_context()) + kwargs["healthchecks"] = list(run_healthchecks()) return super().get_context_data(**kwargs) - def _get_healthcheck_context(self): - return { - "show_system_health": True, - "last_run_periodic_call": LastRunPeriodicCall.get_last_call(), - "last_run_periodic_call_stuck": LastRunPeriodicCall.is_stuck(), - } - class PersonalDataSettingsView(LoginRequiredMixin, TemplateView): template_name = "core/settings/settings_personal_data.html" diff --git a/tests/core/test_views_settings.py b/tests/core/test_views_settings.py index 75b1b4c90..dff60bcfe 100644 --- a/tests/core/test_views_settings.py +++ b/tests/core/test_views_settings.py @@ -10,5 +10,11 @@ def test_settings_calendar(django_app, volunteer): response = django_app.get(reverse("core:settings_calendar"), user=volunteer) calendar_url = response.html.find("input", id="calendar-url")["value"] assert calendar_url + response = django_app.get(calendar_url, user=volunteer) assert response + + +def test_settings_instance(django_app, superuser): + response = django_app.get(reverse("core:settings_instance"), user=superuser) + assert "System health" in response.html.text From da30f27c6216f710ffe1f585010096f731d0e1b8 Mon Sep 17 00:00:00 2001 From: Felix Rindt Date: Wed, 13 Sep 2023 19:55:04 +0200 Subject: [PATCH 12/14] refactor health check --- .dockerignore | 148 ++------------------ deployment/docker/cron.sh | 2 +- ephios/core/services/health/healthchecks.py | 6 +- ephios/core/views/healthcheck.py | 22 +-- 4 files changed, 24 insertions(+), 154 deletions(-) diff --git a/.dockerignore b/.dockerignore index 8978e1860..e980b6d7c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,144 +1,12 @@ -# not execution related files +# not docker execution related files +.github +.pytest_cache +data docs tests - -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# build artifacts -docs/api/ephios-open-api-schema.yml - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -data - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments +dist +build .env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ +.idea +.env.example -# IDEA files -.idea/ diff --git a/deployment/docker/cron.sh b/deployment/docker/cron.sh index a6492d73b..394c8354b 100644 --- a/deployment/docker/cron.sh +++ b/deployment/docker/cron.sh @@ -1,7 +1,7 @@ #!/bin/bash while [ true ]; do - sleep 60 echo "Running cron job" /usr/local/bin/python3 -m ephios run_periodic + sleep 60 done \ No newline at end of file diff --git a/ephios/core/services/health/healthchecks.py b/ephios/core/services/health/healthchecks.py index ea14c22e7..df1e5de53 100644 --- a/ephios/core/services/health/healthchecks.py +++ b/ephios/core/services/health/healthchecks.py @@ -82,7 +82,7 @@ def check(self): if settings.DATABASES["default"]["ENGINE"] == "django.db.backends.sqlite3": return HealthCheckStatus.WARNING, _( - "Using SQLite, this is not recommended in production." + "Using SQLite, which should not be used in production." ) return HealthCheckStatus.OK, _("Database connection established.") @@ -109,7 +109,7 @@ def check(self): == "django.core.cache.backends.locmem.LocMemCache" ): return HealthCheckStatus.WARNING, _( - "Using LocMemCache, this is not recommended in production." + "Using LocMemCache, which should not be used in production." ) return HealthCheckStatus.OK, _("Cache connection established.") @@ -166,7 +166,7 @@ def check(self): ) return ( HealthCheckStatus.OK, - mark_safe(_("Media root writable by application server.")), + mark_safe(_("Media root is writable by application server.")), ) diff --git a/ephios/core/views/healthcheck.py b/ephios/core/views/healthcheck.py index 23310d8bb..156d7069b 100644 --- a/ephios/core/views/healthcheck.py +++ b/ephios/core/views/healthcheck.py @@ -6,19 +6,21 @@ class HealthCheckView(View): def get(self, request, *args, **kwargs): - messages = [] - errors = [] + okays = [] + not_okays = [] for check, status, message in run_healthchecks(): text = f"{check.name}: {message}" if status == HealthCheckStatus.OK: - messages.append(text) + okays.append(text) else: - errors.append(text) + not_okays.append(text) - if errors: - return HttpResponse( - "
".join(errors) + "

" + "
".join(messages), status=503 - ) - - return HttpResponse("
".join(messages), status=200) + status = 200 + message = "" + if not_okays: + status = 503 + message += "NOT OK

" + "
".join(not_okays) + "

" + if okays: + message += "OK

" + "
".join(okays) + "

" + return HttpResponse(message, status=status) From 08bee121db91de0431921bec7bcbcce83b5afda7 Mon Sep 17 00:00:00 2001 From: Felix Rindt Date: Wed, 13 Sep 2023 20:03:54 +0200 Subject: [PATCH 13/14] link to ghcr --- .github/workflows/docker.yml | 3 --- docs/admin/deployment/docker/index.rst | 8 +++++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5d6116abd..7cdd90854 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -7,9 +7,6 @@ on: - 'main' tags: - 'v*' - pull_request: - branches: - - 'main' jobs: docker: diff --git a/docs/admin/deployment/docker/index.rst b/docs/admin/deployment/docker/index.rst index de32fe1dd..cd61a4548 100644 --- a/docs/admin/deployment/docker/index.rst +++ b/docs/admin/deployment/docker/index.rst @@ -10,7 +10,13 @@ database, redis and some ssl-terminating webserver/proxy to be set up. Also, static files need to be served by a webserver from a common volume. -The image is available on the Github Container Registry. +The image is available on the Github Container Registry under +`ghcr.io/ephios-dev/ephios `_. + +The image is based on the official python image. Tags are ``main`` for the latest commit +on the main branch, ``latest`` for the latest release and ``v0.x.y`` for a specific +release. + Deployment with Docker Compose ------------------------------ From 4c10147cd7567f25a92df4909c98dfe827a5b359 Mon Sep 17 00:00:00 2001 From: Felix Rindt Date: Thu, 14 Sep 2023 16:11:26 +0200 Subject: [PATCH 14/14] move some settings to checks and build commands --- .github/workflows/docker.yml | 2 +- .github/workflows/tests.yml | 4 +- deployment/compose/docker-compose.yaml | 3 +- deployment/docker/entrypoint.sh | 4 +- docs/admin/deployment/manual/index.rst | 27 ++++------ docs/development/contributing.rst | 2 +- ephios/core/apps.py | 3 -- ephios/core/checks.py | 53 ++++++++++++++----- ephios/{extra => core}/management/__init__.py | 0 .../management/commands/__init__.py | 0 ephios/core/management/commands/build.py | 11 ++++ .../management/commands/devdata.py | 0 .../management/commands/generate_vapid_key.py | 20 +++++++ .../management/commands/run_periodic.py | 2 + .../management/commands/send_notifications.py | 2 + ephios/core/plugins.py | 2 - ephios/core/services/health/healthchecks.py | 41 ++++++++++---- ephios/core/views/healthcheck.py | 2 +- ephios/extra/secrets.py | 1 + ephios/settings.py | 35 ++++++------ 20 files changed, 138 insertions(+), 76 deletions(-) rename ephios/{extra => core}/management/__init__.py (100%) rename ephios/{extra => core}/management/commands/__init__.py (100%) create mode 100644 ephios/core/management/commands/build.py rename ephios/{extra => core}/management/commands/devdata.py (100%) create mode 100644 ephios/core/management/commands/generate_vapid_key.py rename ephios/{extra => core}/management/commands/run_periodic.py (90%) rename ephios/{extra => core}/management/commands/send_notifications.py (73%) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 7cdd90854..87aa939c8 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -35,6 +35,6 @@ jobs: uses: docker/build-push-action@v5 with: context: . - push: true #${{ github.event_name != 'pull_request' }} + push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 53a134f2d..fb6c8e301 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -78,9 +78,7 @@ jobs: - name: Prepare files for test run run: | cp .env.example .env - poetry run python manage.py compilemessages - poetry run python manage.py collectstatic - poetry run python manage.py compilejsi18n + poetry run python manage.py build - name: Setup postgres # postgres 14 on ubuntu 22.04 run: | diff --git a/deployment/compose/docker-compose.yaml b/deployment/compose/docker-compose.yaml index 94f176302..7de7b4acc 100644 --- a/deployment/compose/docker-compose.yaml +++ b/deployment/compose/docker-compose.yaml @@ -1,6 +1,7 @@ services: ephios: - build: ../../ + # to build locally use `build: "../../"` instead of `image: ...` + image: ghcr.io/ephios-dev/ephios:latest restart: unless-stopped environment: DEBUG: "False" diff --git a/deployment/docker/entrypoint.sh b/deployment/docker/entrypoint.sh index 325e425bc..7dd241efa 100644 --- a/deployment/docker/entrypoint.sh +++ b/deployment/docker/entrypoint.sh @@ -4,9 +4,7 @@ set -e if [ "$1" == "run" ]; then python manage.py migrate - python manage.py collectstatic --no-input - python manage.py compilemessages - python manage.py compilejsi18n + python manage.py build exec supervisord -n -c /etc/supervisord.conf fi diff --git a/docs/admin/deployment/manual/index.rst b/docs/admin/deployment/manual/index.rst index db794f7e2..30850baf4 100644 --- a/docs/admin/deployment/manual/index.rst +++ b/docs/admin/deployment/manual/index.rst @@ -60,13 +60,6 @@ The reverse proxy needs to be able to read the static files stored in ``/var/eph .. _web_push_notifications: -VAPID keys -'''''''''' - -ephios uses `VAPID `_ to send push notifications. -Vapid keys will be generated automatically if they are not present. -If you want to use your own keys, you can generate them with ``/home/ephios/venv/bin/vapid --gen``. - Config file ''''''''''' @@ -111,9 +104,7 @@ Now that the configuration is in place, we can build the static files and the tr $ export ENV_PATH="/home/ephios/ephios.env" $ source /home/ephios/venv/bin/activate $ python -m ephios migrate - $ python -m ephios collectstatic --noinput - $ python -m ephios compilemessages - $ python -m ephios compilejsi18n + $ python -m ephios build Setup cron '''''''''' @@ -247,12 +238,16 @@ You can now create your first user account by running: You should now secure your installation. Try starting with the tips below. -To install a plugin install them via pip and restart the ephios-gunicorn service: +To install a plugin install them via pip: .. code-block:: console - # ENV_PATH="/home/ephios/ephios.env" sudo -u ephios /home/ephios/venv/bin/pip install ephios- - # systemctl restart ephios-gunicorn + # sudo -u ephios -i + $ export ENV_PATH="/home/ephios/ephios.env" + $ source /home/ephios/venv/bin/activate + $ pip install "ephios-" + $ python -m ephios migrate + $ python -m ephios build To update ephios create a backup of your database and files and run: @@ -263,11 +258,9 @@ To update ephios create a backup of your database and files and run: $ source /home/ephios/venv/bin/activate $ pip install -U "ephios[redis,pgsql]" $ python -m ephios migrate - $ python -m ephios collectstatic --noinput - $ python -m ephios compilemessages - $ python -m ephios compilejsi18n + $ python -m ephios build -Then, as root, restart the gunicorn service: +After installing plugins or updating, restart the gunicorn service: .. code-block:: console diff --git a/docs/development/contributing.rst b/docs/development/contributing.rst index ae1069ef1..fc82fd725 100644 --- a/docs/development/contributing.rst +++ b/docs/development/contributing.rst @@ -29,7 +29,7 @@ To set up a development version on your local machine, you need to execute the f #. Install dependencies with ``poetry install`` #. Create env file with ``cp .env.example .env`` #. Migrate the database with ``python manage.py migrate`` -#. Compile translations with ``python manage.py compilemessages`` and ``python manage.py compilejsi18n`` +#. Prepare files for the installation with ``python manage.py build`` #. Load data for testing with ``python manage.py devdata`` #. Start the development server with ``python manage.py runserver`` #. Open your web browser, visit ``http://localhost:8000`` and log in with the default credentials (user ``admin@localhost`` and password ``admin``) diff --git a/ephios/core/apps.py b/ephios/core/apps.py index 9db8a1259..408828ea6 100644 --- a/ephios/core/apps.py +++ b/ephios/core/apps.py @@ -1,7 +1,6 @@ import logging from django.apps import AppConfig -from django.conf import settings from dynamic_preferences.registries import preference_models logger = logging.getLogger(__name__) @@ -18,5 +17,3 @@ def ready(self): EventTypePreference = self.get_model("EventTypePreference") preference_models.register(EventTypePreference, event_type_preference_registry) - - logger.info("Installed plugins: %s", ", ".join(settings.PLUGINS)) diff --git a/ephios/core/checks.py b/ephios/core/checks.py index f6050bf61..b182beabb 100644 --- a/ephios/core/checks.py +++ b/ephios/core/checks.py @@ -1,27 +1,52 @@ import os from django.conf import settings +from django.core.checks import Error as DjangoError from django.core.checks import Warning as DjangoWarning from django.core.checks import register @register("ephios", deploy=True) -def check_data_dir_is_not_inside_base_dir(app_configs, **kwargs): - """ - On production setups, the data directory should - not be inside the base directory (ephios package directory). - """ +def check_ephios_deploy_settings(app_configs, **kwargs): + # On production setups, the data directory should + # not be inside the base directory (ephios package directory). errors = [] + + bad_directories = [] for name, directory in settings.DIRECTORIES.items(): if os.path.commonpath([settings.BASE_DIR, directory]) == settings.BASE_DIR: - errors.append( - DjangoWarning( - "ephios data is stored near the ephios package directory.", - hint=f"You probably don't want to have {name} at {directory} to " - f"be inside or next to the ephios python package at " - f"{settings.BASE_DIR}! " - f"Configure ephios to use another location.", - id="ephios.W001", - ) + bad_directories.append((name, directory)) + if bad_directories: + errors.append( + DjangoWarning( + "ephios data is stored near the ephios package directory.", + hint=f"You probably don't want to have the {', '.join(name for name, _ in bad_directories)} " + f"be inside or next to the ephios python package at " + f"{settings.BASE_DIR}! " + f"Configure ephios to use another location.", + id="ephios.W001", + ) + ) + + # Check that VAPID keys were used to configure webpush. + if not settings.WEBPUSH_SETTINGS: + errors.append( + DjangoWarning( + "WEBPUSH_SETTINGS are not configured.", + hint="You need to generate a vapid key using ./manage.py generate_vapid_key in order to " + "use push notifications.", + id="ephios.W002", ) + ) + + if not os.access(settings.MEDIA_ROOT, os.W_OK): + errors.append( + DjangoError( + "Media root not writable by application server.", + hint=f"You need to make sure that the application server can write to the media root at " + f"{settings.MEDIA_ROOT}.", + id="ephios.E001", + ) + ) + return errors diff --git a/ephios/extra/management/__init__.py b/ephios/core/management/__init__.py similarity index 100% rename from ephios/extra/management/__init__.py rename to ephios/core/management/__init__.py diff --git a/ephios/extra/management/commands/__init__.py b/ephios/core/management/commands/__init__.py similarity index 100% rename from ephios/extra/management/commands/__init__.py rename to ephios/core/management/commands/__init__.py diff --git a/ephios/core/management/commands/build.py b/ephios/core/management/commands/build.py new file mode 100644 index 000000000..63e92cb64 --- /dev/null +++ b/ephios/core/management/commands/build.py @@ -0,0 +1,11 @@ +from django.core.management import BaseCommand, call_command + + +class Command(BaseCommand): + help = "(Re)build static files, language files, directories and VAPID key" + + def handle(self, *args, **options): + call_command("generate_vapid_key", verbosity=1) + call_command("collectstatic", verbosity=1, interactive=False) + call_command("compilemessages", verbosity=1) + call_command("compilejsi18n", verbosity=1) diff --git a/ephios/extra/management/commands/devdata.py b/ephios/core/management/commands/devdata.py similarity index 100% rename from ephios/extra/management/commands/devdata.py rename to ephios/core/management/commands/devdata.py diff --git a/ephios/core/management/commands/generate_vapid_key.py b/ephios/core/management/commands/generate_vapid_key.py new file mode 100644 index 000000000..211e93455 --- /dev/null +++ b/ephios/core/management/commands/generate_vapid_key.py @@ -0,0 +1,20 @@ +import os.path + +from django.conf import settings +from django.core.management import BaseCommand +from py_vapid import Vapid + + +class Command(BaseCommand): + help = "Generate VAPID key at VAPID_PRIVATE_KEY_PATH if it does not exist" + + def handle(self, *args, **options): + if not os.path.exists(settings.VAPID_PRIVATE_KEY_PATH): + print("Generating VAPID key") + vapid = Vapid() + vapid.generate_keys() + vapid.save_key(settings.VAPID_PRIVATE_KEY_PATH) + vapid.save_public_key(settings.VAPID_PRIVATE_KEY_PATH + ".pub") + print(f"Saved to {settings.VAPID_PRIVATE_KEY_PATH}") + else: + print(f"VAPID key already exists at {settings.VAPID_PRIVATE_KEY_PATH}") diff --git a/ephios/extra/management/commands/run_periodic.py b/ephios/core/management/commands/run_periodic.py similarity index 90% rename from ephios/extra/management/commands/run_periodic.py rename to ephios/core/management/commands/run_periodic.py index 01b00965f..7d33375ed 100644 --- a/ephios/extra/management/commands/run_periodic.py +++ b/ephios/core/management/commands/run_periodic.py @@ -8,6 +8,8 @@ class Command(BaseCommand): + help = "Run periodic tasks" + def handle(self, *args, **options): logger.info("Running periodic tasks") periodic_signal.send(self) diff --git a/ephios/extra/management/commands/send_notifications.py b/ephios/core/management/commands/send_notifications.py similarity index 73% rename from ephios/extra/management/commands/send_notifications.py rename to ephios/core/management/commands/send_notifications.py index cc1bca385..cd3c2899b 100644 --- a/ephios/extra/management/commands/send_notifications.py +++ b/ephios/core/management/commands/send_notifications.py @@ -4,5 +4,7 @@ class Command(BaseCommand): + help = "Send all notifications (for testing, use run_periodic in production)" + def handle(self, *args, **options): send_all_notifications() diff --git a/ephios/core/plugins.py b/ephios/core/plugins.py index 3140b8046..d070e99fb 100644 --- a/ephios/core/plugins.py +++ b/ephios/core/plugins.py @@ -7,8 +7,6 @@ logger = logging.getLogger(__name__) -# The plugin mechanics are heavily inspired by pretix (then licenced under Apache 2.0) - Check it out! - def get_all_plugins(): """ diff --git a/ephios/core/services/health/healthchecks.py b/ephios/core/services/health/healthchecks.py index df1e5de53..0766e9924 100644 --- a/ephios/core/services/health/healthchecks.py +++ b/ephios/core/services/health/healthchecks.py @@ -1,10 +1,10 @@ import os -from pathlib import Path from django.conf import settings from django.contrib.auth.models import Permission from django.contrib.humanize.templatetags.humanize import naturaltime from django.dispatch import receiver +from django.template.defaultfilters import floatformat from django.utils.safestring import mark_safe from django.utils.translation import gettext_lazy as _ @@ -149,24 +149,45 @@ def check(self): ) -class WritableMediaRootHealthCheck(AbstractHealthCheck): - slug = "writable_media_root" - name = _("Writable Media Root") - description = _("The media root must be writable by the application server.") +class DiskSpaceHealthCheck(AbstractHealthCheck): + slug = "disk_space" + name = _("Disk space") + description = _("Disk space needs to be available to store data.") documentation_link = ( "https://docs.ephios.de/en/stable/admin/deployment/manual/index.html#data-directory" ) def check(self): - media_root = Path(settings.MEDIA_ROOT) - if not os.access(media_root, os.W_OK): + # if under 100 MB are available, we consider this an error + # if under 1 GB are available, we consider this a warning + # otherwise, we consider this ok + disk_usage = os.statvfs(settings.MEDIA_ROOT) + free_bytes = disk_usage.f_bavail * disk_usage.f_frsize + MEGA = 1024 * 1024 + if free_bytes < 100 * MEGA: return ( HealthCheckStatus.ERROR, - mark_safe(_("Media root not writable by application server.")), + mark_safe( + _( + "Less than 100 MB of disk space available. " + "Please free up some disk space." + ) + ), + ) + if free_bytes < 1024 * MEGA: + return ( + HealthCheckStatus.WARNING, + mark_safe( + _("Less than 1 GB of disk space available. Please free up some disk space.") + ), ) return ( HealthCheckStatus.OK, - mark_safe(_("Media root is writable by application server.")), + mark_safe( + _("{disk_space} of disk space available.").format( + disk_space=f"{floatformat(free_bytes / MEGA / 1024,1)} GB" + ) + ), ) @@ -175,4 +196,4 @@ def register_core_healthchecks(sender, **kwargs): yield DBHealthCheck yield CacheHealthCheck yield CronJobHealthCheck - yield WritableMediaRootHealthCheck + yield DiskSpaceHealthCheck diff --git a/ephios/core/views/healthcheck.py b/ephios/core/views/healthcheck.py index 156d7069b..358b6c285 100644 --- a/ephios/core/views/healthcheck.py +++ b/ephios/core/views/healthcheck.py @@ -11,7 +11,7 @@ def get(self, request, *args, **kwargs): for check, status, message in run_healthchecks(): text = f"{check.name}: {message}" - if status == HealthCheckStatus.OK: + if status in {HealthCheckStatus.OK, HealthCheckStatus.WARNING}: okays.append(text) else: not_okays.append(text) diff --git a/ephios/extra/secrets.py b/ephios/extra/secrets.py index 9ca3a2cd5..c93d37efb 100644 --- a/ephios/extra/secrets.py +++ b/ephios/extra/secrets.py @@ -10,6 +10,7 @@ def django_secret_from_file(path: str): return f.read().strip() chars = string.ascii_letters + string.digits + string.punctuation secret = get_random_string(50, chars) + os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "w", encoding="utf8") as f: os.chmod(path, 0o600) try: diff --git a/ephios/settings.py b/ephios/settings.py index a3af39dcd..218f2f901 100644 --- a/ephios/settings.py +++ b/ephios/settings.py @@ -47,10 +47,8 @@ "LOG_DIR": LOG_DIR, "MEDIA_ROOT": MEDIA_ROOT, } - -for path in DIRECTORIES.values(): - if not os.path.exists(path): - os.makedirs(path, exist_ok=True) +for directory in DIRECTORIES.values(): + os.makedirs(directory, exist_ok=True) if "SECRET_KEY" in env: SECRET_KEY = env.str("SECRET_KEY") @@ -385,22 +383,19 @@ def GET_SITE_URL(): VAPID_PRIVATE_KEY_PATH = env.str( "VAPID_PRIVATE_KEY_PATH", os.path.join(PRIVATE_DIR, "vapid_key.pem") ) -if not os.path.exists(VAPID_PRIVATE_KEY_PATH): - vapid = Vapid() - vapid.generate_keys() - vapid.save_key(VAPID_PRIVATE_KEY_PATH) - vapid.save_public_key(VAPID_PRIVATE_KEY_PATH + ".pub") - -vp = Vapid().from_file(VAPID_PRIVATE_KEY_PATH) -WEBPUSH_SETTINGS = { - "VAPID_PUBLIC_KEY": b64urlencode( - vp.public_key.public_bytes( - serialization.Encoding.X962, serialization.PublicFormat.UncompressedPoint - ) - ), - "VAPID_PRIVATE_KEY": vp, - "VAPID_ADMIN_EMAIL": ADMINS[0][1], -} +WEBPUSH_SETTINGS = {} +if os.path.exists(VAPID_PRIVATE_KEY_PATH): + vp = Vapid().from_file(VAPID_PRIVATE_KEY_PATH) + WEBPUSH_SETTINGS = { + "VAPID_PUBLIC_KEY": b64urlencode( + vp.public_key.public_bytes( + serialization.Encoding.X962, serialization.PublicFormat.UncompressedPoint + ) + ), + "VAPID_PRIVATE_KEY": vp, + "VAPID_ADMIN_EMAIL": ADMINS[0][1], + } + # Health check # interval for calls to the run_periodic_tasks management command over which the cronjob is considered to be broken