diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..e980b6d7c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +# not docker execution related files +.github +.pytest_cache +data +docs +tests +dist +build +.env +.idea +.env.example + diff --git a/.env.example b/.env.example index 7806545ab..a1777254d 100644 --- a/.env.example +++ b/.env.example @@ -1,15 +1,8 @@ -SECRET_KEY=b0jbk58)l&^hy5cht+j$n9!laoke5ivdc+3&@qf2yrt1udvb85 DEBUG=True -DEBUG_TOOLBAR=False DATABASE_URL=sqlite:///data/db.sqlite3 ALLOWED_HOSTS="*" -STATIC_URL=/static/ -STATIC_ROOT=data/static/ -LOGGING_FILE=data/logs/ephios.log -INTERNAL_IPS="127.0.0.1" SITE_URL=http://localhost:8000 -EMAIL_URL=dummymail:// +EMAIL_URL=consolemail:// DEFAULT_FROM_EMAIL=webmaster@localhost SERVER_EMAIL=root@localhost -ADMINS=Root User -# VAPID_PRIVATE_KEY_PATH=data/private_key.pem # push notifications, create in `data/` with `vapid --gen` +ADMINS="Root User " diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 000000000..87aa939c8 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,40 @@ + +name: docker +on: + workflow_dispatch: + push: + branches: + - 'main' + tags: + - 'v*' + +jobs: + docker: + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/ephios-dev/ephios + + - name: Login to Github Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 990fd7dd1..59f450a87 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,10 +1,10 @@ -name: Publish to pypi +name: Publish to PyPI on: push: tags: - 'v*.*.*' jobs: - build: + pypi: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 53a134f2d..fb6c8e301 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -78,9 +78,7 @@ jobs: - name: Prepare files for test run run: | cp .env.example .env - poetry run python manage.py compilemessages - poetry run python manage.py collectstatic - poetry run python manage.py compilejsi18n + poetry run python manage.py build - name: Setup postgres # postgres 14 on ubuntu 22.04 run: | diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..a41842a33 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,42 @@ +# syntax=docker/dockerfile:1 +FROM python:3.11-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 +ENV POETRY_VIRTUALENVS_CREATE=false +ENV POETRY_VERSION=1.6.1 + +WORKDIR /usr/src/ephios + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + gettext \ + supervisor \ + locales && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN dpkg-reconfigure locales && \ + locale-gen C.UTF-8 && \ + /usr/sbin/update-locale LANG=C.UTF-8 + +RUN pip install "poetry==$POETRY_VERSION" gunicorn +RUN poetry self add "poetry-dynamic-versioning[plugin]" + +RUN mkdir -p /var/ephios/data/ && \ + mkdir -p /var/log/supervisord/ && \ + mkdir -p /var/run/supervisord/ + +COPY . /usr/src/ephios +RUN poetry install --all-extras --without=dev + +COPY deployment/docker/entrypoint.sh /usr/local/bin/ephios +RUN chmod +x /usr/local/bin/ephios + +COPY deployment/docker/supervisord.conf /etc/supervisord.conf +COPY deployment/docker/cron.sh /usr/local/bin/cron.sh +RUN chmod +x /usr/local/bin/cron.sh + +ENTRYPOINT ["ephios"] +CMD ["run"] +HEALTHCHECK CMD curl -f http://localhost:80/healthcheck || exit 1 diff --git a/deployment/compose/docker-compose.yaml b/deployment/compose/docker-compose.yaml new file mode 100644 index 000000000..7de7b4acc --- /dev/null +++ b/deployment/compose/docker-compose.yaml @@ -0,0 +1,50 @@ +services: + ephios: + # to build locally use `build: "../../"` instead of `image: ...` + image: ghcr.io/ephios-dev/ephios:latest + restart: unless-stopped + environment: + DEBUG: "False" + TRUST_X_FORWARDED_PROTO: "True" + DATA_DIR: "/var/ephios/data/" + DATABASE_URL: "postgres://ephios:ephios@ephios_postgres/ephios" + CACHE_URL: "redis://ephios_redis/0" + ALLOWED_HOSTS: "*" + # change the following to your needs + EMAIL_URL: "consolemail://" + SITE_URL: "http://localhost" + DEFAULT_FROM_EMAIL: "webmaster@localhost" + SERVER_EMAIL: "root@localhost" + ADMINS: "Root User " + volumes: + - ephios_django_data:/var/ephios/data/ + + ephios_redis: + image: redis:7-alpine + command: redis-server + restart: unless-stopped + + ephios_postgres: + image: postgres:12 + restart: unless-stopped + environment: + POSTGRES_DB: ephios + POSTGRES_USER: ephios + POSTGRES_PASSWORD: ephios + volumes: + - ephios_postgres_data:/var/lib/postgresql/data + + ephios_nginx: + image: nginx:1.19 + restart: unless-stopped + ports: + - 80:80 + volumes: + - ./nginx.conf:/etc/nginx/conf.d/default.conf + - ephios_django_data:/var/ephios/data/ + depends_on: + - ephios + +volumes: + ephios_django_data: { } + ephios_postgres_data: { } diff --git a/deployment/compose/nginx.conf b/deployment/compose/nginx.conf new file mode 100644 index 000000000..304922765 --- /dev/null +++ b/deployment/compose/nginx.conf @@ -0,0 +1,19 @@ +server { + listen 80 default_server; + listen [::]:80 ipv6only=on default_server; + + location / { + proxy_pass http://ephios:80; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Host $http_host; + proxy_redirect off; + } + + location /static/ { + alias /var/ephios/data/public/static/; + access_log off; + expires 1d; + add_header Cache-Control "public"; + } +} diff --git a/deployment/docker/cron.sh b/deployment/docker/cron.sh new file mode 100644 index 000000000..394c8354b --- /dev/null +++ b/deployment/docker/cron.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +while [ true ]; do + echo "Running cron job" + /usr/local/bin/python3 -m ephios run_periodic + sleep 60 +done \ No newline at end of file diff --git a/deployment/docker/entrypoint.sh b/deployment/docker/entrypoint.sh new file mode 100644 index 000000000..7dd241efa --- /dev/null +++ b/deployment/docker/entrypoint.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e + +if [ "$1" == "run" ]; then + python manage.py migrate + python manage.py build + exec supervisord -n -c /etc/supervisord.conf +fi + +exec python manage.py "$@" diff --git a/deployment/docker/supervisord.conf b/deployment/docker/supervisord.conf new file mode 100644 index 000000000..dfcc8e9ae --- /dev/null +++ b/deployment/docker/supervisord.conf @@ -0,0 +1,23 @@ +[supervisord] +nodaemon=true +user=root +logfile=/var/log/supervisord/supervisord.log +pidfile=/var/run/supervisord/supervisord.pid +childlogdir=/var/log/supervisord/ +logfile_maxbytes=50MB +logfile_backups=10 +loglevel=error + +[program:gunicorn] +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 +command=gunicorn ephios.wsgi --bind 0.0.0.0:80 --name ephios --workers 5 --max-requests 1000 --max-requests-jitter 100 + +[program:cronscript] +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 +command=/usr/local/bin/cron.sh diff --git a/docs/admin/configuration/index.rst b/docs/admin/configuration/index.rst index 3997714fa..c54260fae 100644 --- a/docs/admin/configuration/index.rst +++ b/docs/admin/configuration/index.rst @@ -10,71 +10,110 @@ for a more in-depth explanation of what they do. .. _env_file_options: -The following variables are available (plugins and some niche features might require additional environment variables): +The following variables are available (plugins and some niche features might require additional environment variables). + +Setup +----- `ENV_PATH`: Path to an environment file. Defaults to `.env` in the location of the ephios package. We recommend setting most of the following variables in this file. -`SECRET_KEY`: - **Required**. Django secret key used to encrypt session data and other sensitive information. +`DJANGO_SETTINGS_MODULE`: + Defaults to `ephios.settings`. If you want to use your own settings file, + set this to the path to your settings file. This variable cannot be set in the environment file. + +Debugging +--------- `DEBUG`: **Required**. Set to `True` to enable debug mode. Must be `False` in production. -`ALLOWED_HOSTS`: - **Required**. Comma-separated list of hostnames that are allowed to access the ephios instance. +`DEBUG_TOOLBAR`: + Set to `True` to enable the django debug toolbar. Must be `False` in production. + Defaults to `False`. -`DATABASE_URL`: - **Required**. URL to the database. See - `django-environ `__ for details. +`INTERNAL_IPS`: + Comma-separated list of IP addresses that are allowed to access the debug toolbar. + Defaults to `127.0.0.1`. -`SITE_URL`: - **Required**. URL used to construct absolute URLs in emails and other places. +Secrets +------- -`CACHE_URL`: - URL to the cache. See - `django-environ `__ for details. +`SECRET_KEY`: + **Important**. Django secret key used to encrypt session data and other sensitive information. + Defaults to a random value persisted into `PRIVATE_DIR/.secret`. -`STATIC_ROOT`: - **Required**: Path where static files are collected to. - A reverse proxy should be configured to serve them at `STATIC_URL`. +`VAPID_PRIVATE_KEY_PATH`: + Path to the private key used to sign web push notifications. If not provided, web push notifications wont work + on some platforms. See :ref:`web_push_notifications` for details. + Defaults to `PRIVATE_DIR/vapid_key.pem`. A keypair is automatically generated if it does not exist. -`EMAIL_URL`: - **Required**. URL to the email smtp server. See - `django-environ `__ for details. +Data storage and Logging +------------------------ -`DEFAULT_FROM_EMAIL`: - **Required**. Email address that is used as the sender for all - emails sent by ephios. (`Django docs `__) +`DATA_DIR`: + **Important**. Base path where ephios defaults to store files. + Defaults to a `data` folder in the location of the ephios package, + which is not recommended for production use. -`SERVER_EMAIL`: - **Required**. Email address that is used as the sender for all - error emails sent by django. (`Django docs `__) +`PUBLIC_DIR`: + Path where public files are stored. Defaults to `DATA_DIR/public`. -`ADMINS`: - **Required**. Email addresses that receive error emails. +`STATIC_ROOT`: + Path where static files (css/js) are collected to. + A reverse proxy should be configured to serve them at `STATIC_URL`. + Defaults to `PUBLIC_DIR/static`. + +`PRIVATE_DIR`: + Path where private files are stored. Defaults to `DATA_DIR/private`. + Make sure access to this folder is restricted to the user running ephios. + +`MEDIA_ROOT`: + Path where uploaded files are stored. + Defaults to `PRIVATE_DIR/media`. + You should backup this folder regularly. -`LOGGING_FILE`: - Path to the log file. If provided, ephios logs to this file. - The file is rotated daily. +`LOG_DIR`: + Path to the folder where log files are put. Files inside are rotated daily. + Defaults to `PRIVATE_DIR/logs`. `LOGGING_BACKUP_DAYS`: Number of days to keep log files. Defaults to 14. + +Database and Caching +-------------------- + +`DATABASE_URL`: + **Required**. URL to the database. See + `django-environ `__ for details. + +`CONN_MAX_AGE`: + Number of seconds to keep database connections open. Defaults to 0, meaning connections are closed after each request. + Refer to the `django docs `__ for details. + +`CACHE_URL`: + URL to the cache. We recommend redis. See + `django-environ `__ for details. + +URLs and Routing +---------------- + +`ALLOWED_HOSTS`: + **Required**. Comma-separated list of hostnames that are allowed to access the ephios instance. + + +`SITE_URL`: + **Required**. URL used to construct absolute URLs in emails and other places. + `STATIC_URL`: URL where the static files can be found by browsers. Defaults to ``/static/``, meaning they are served by the same host. -`DEBUG_TOOLBAR`: - Set to `True` to enable the django debug toolbar. Must be `False` in production. +Security +-------- -`INTERNAL_IPS`: - Comma-separated list of IP addresses that are allowed to access the debug toolbar. - -`VAPID_PRIVATE_KEY_PATH`: - Path to the private key used to sign web push notifications. If not provided, web push notifications wont work - on some platforms. See :ref:`web_push_notifications` for details. `SECURE_HSTS_SECONDS`: Number of seconds to set the `Strict-Transport-Security `__ @@ -88,10 +127,29 @@ The following variables are available (plugins and some niche features might req Set the `preload `__ flag in the `Strict-Transport-Security `__ header. Defaults to `False`. -`CONN_MAX_AGE`: - Number of seconds to keep database connections open. Defaults to 0, meaning connections are closed after each request. - Refer to the `django docs `__ for details. +`TRUST_X_FORWARDED_PROTO`: + ephios must be served over HTTPS in production. In some setups, ephios is behind a reverse proxy that terminates + SSL connections and the Origin header is not set with a https scheme. In this case, the proxy can communicate + the fact that the connection is secure by setting the + `X-Forwarded-Proto `__ header. + Then this setting must be set to `True`. See + `django docs `__ + for details. Defaults to `False`. -`DJANGO_SETTINGS_MODULE`: - Defaults to `ephios.settings`. If you want to use your own settings file, - set this to the path to your settings file. This variable cannot be set in the environment file. +E-Mail +------ + +`EMAIL_URL`: + **Required**. URL to the email smtp server. See + `django-environ `__ for details. + +`DEFAULT_FROM_EMAIL`: + **Required**. Email address that is used as the sender for all + emails sent by ephios. (`Django docs `__) + +`SERVER_EMAIL`: + **Required**. Email address that is used as the sender for all + error emails sent by django. (`Django docs `__) + +`ADMINS`: + **Required**. Email addresses that receive error emails. diff --git a/docs/admin/deployment/docker/index.rst b/docs/admin/deployment/docker/index.rst new file mode 100644 index 000000000..cd61a4548 --- /dev/null +++ b/docs/admin/deployment/docker/index.rst @@ -0,0 +1,61 @@ +Docker +====== + +Docker Image +------------ + +We automatically build a Docker image for every release. +It uses gunicorn as a WSGI-server and still requires a +database, redis and some ssl-terminating webserver/proxy +to be set up. Also, static files need to be served by +a webserver from a common volume. + +The image is available on the Github Container Registry under +`ghcr.io/ephios-dev/ephios `_. + +The image is based on the official python image. Tags are ``main`` for the latest commit +on the main branch, ``latest`` for the latest release and ``v0.x.y`` for a specific +release. + + +Deployment with Docker Compose +------------------------------ + +We provide a basic docker-compose file that adds +nginx, postgres and redis to the mix and exposes +the app on port 80. You still need to either provide +an SSL-terminating proxy in front of it or +configure the nginx container to do so. + +The compose file can be found at +``deployment/compose/docker-compose.yml`` and +`on github `_. +Feel free to use it as a starting point for your own deployment. + +The container defines two volumes for the database and +ephios data files. You should mount them to a persistent +location on your host. + +Make sure to change the environment variables in the +compose file to your needs. Have a look at +the :ref:`configuration options `. + +To start the compose file and add a first superuser run: + +.. code-block:: console + + # cd deployment/compose/ + # docker compose up --build + # docker exec -it compose-ephios-1 python -m ephios createsuperuser + +If you want to test the container without https, that's +only possible by changing it the compose file: + +.. code-block:: yaml + + DEBUG: "True" + TRUST_X_FORWARDED_PROTO: "False" + +The first line enables debug mode and therefore disables a redirect from http to https. +The second line disables trusting the X-Forwarded-Proto header set to https by the nginx +container, which is required for djangos CSRF protection to not kick in on http requests. \ No newline at end of file diff --git a/docs/admin/deployment/index.rst b/docs/admin/deployment/index.rst index 303707bd2..4938dc91e 100644 --- a/docs/admin/deployment/index.rst +++ b/docs/admin/deployment/index.rst @@ -3,11 +3,6 @@ Deploying ephios This section shows how to deploy ephios in a production environment. - -.. toctree:: - :maxdepth: 2 - - Prerequisites ------------- @@ -28,289 +23,15 @@ To run ephios (a django project) in production, you generally need: Installation ------------ -Generally, ephios can be installed like most django projects. We plan on providing a docker image in the future. - -Manual installation -~~~~~~~~~~~~~~~~~~~ - -To run ephios on a debian-based system, make sure you have installed the prerequisites mentioned above. -What follows is a rough guide to install ephios and configure the supporting services. -Feel free to adapt it to your needs and style. The guide assumes you are logged in as root -(``#`` is a root prompt, ``$`` the ephios user prompt). - -Unix user -''''''''' - -Create a unix user that will run the ephios. You should avoid running ephios with root privileges. - -.. code-block:: console - - # adduser --disabled-password --home /home/ephios ephios - -Package dependencies -'''''''''''''''''''' - -Install the system packages that ephios depends on. - -.. code-block:: console - - # apt-get install gettext - -python environment and ephios package -''''''''''''''''''''''''''''''''''''' - -Create a `virtualenv `_ for ephios and install the ephios package. -Replace pgsql with mysql if you want to use mysql. - -.. code-block:: console - - # sudo -u ephios python3.11 -m venv /home/ephios/venv - # sudo -u ephios /home/ephios/venv/bin/pip install gunicorn "ephios[redis,pgsql]" - -Database -'''''''' - -Create a database user and a database for ephios. For postgres this could look like this: - -.. code-block:: console - - # sudo -u postgres createuser ephios - # sudo -u postgres createdb -O ephios ephios - -Make sure the encoding of the database is UTF-8. - -Data directory -'''''''''''''' - -ephios stores some data in the file system. Create the folders and make sure the ephios user can write to them. -The reverse proxy needs to be able to read the static files stored in there. - -.. code-block:: console - - # mkdir -p /var/ephios/data/static - # chown -R ephios:ephios /var/ephios - -.. _web_push_notifications: - -VAPID keys -'''''''''' - -ephios uses `VAPID `_ to send push notifications. Create a VAPID key pair: - -.. code-block:: console - - # sudo -u ephios mkdir -p /home/ephios/vapid - # cd /home/ephios/vapid - # sudo -u ephios /home/ephios/venv/bin/vapid --gen - -Config file -''''''''''' - -ephios can be configured using environment variables. They can also be read from a file. -Create a file ``/home/ephios/ephios.env`` (owned by the ephios user) with the following -content, replacing the values with your own: - -.. code-block:: - - SECRET_KEY= - DATABASE_URL=psql://dbuser:dbpass@localhost:5432/ephios - ALLOWED_HOSTS="your.domain.org" - SITE_URL=https://your.domain.org - EMAIL_URL=smtp+ssl://emailuser:emailpass@smtp.domain.org:465 - DEFAULT_FROM_EMAIL=ephios@domain.org - SERVER_EMAIL=ephios@domain.org - ADMINS=Org Admin - VAPID_PRIVATE_KEY_PATH=/home/ephios/vapid/private_key.pem - CACHE_URL="redis://127.0.0.1:6379/1" - STATIC_ROOT=/var/ephios/data/static/ - LOGGING_FILE=/var/ephios/data/logs/ephios.log - DEBUG=False - - -For details on the configuration options and syntax, see :ref:`configuration options `. - -To test your configuration, run: - -.. code-block:: console - - # sudo -u ephios -i - $ export ENV_PATH="/home/ephios/ephios.env" - $ source /home/ephios/venv/bin/activate - $ python -m ephios check --deploy - $ python -m ephios sendtestemail --admin - -Build ephios files -'''''''''''''''''' - -Now that the configuration is in place, we can build the static files and the translation files. +Generally, ephios can be installed like most django projects. +We prepared some guides for common deployment scenarios: -.. code-block:: console - - # sudo -u ephios -i - $ export ENV_PATH="/home/ephios/ephios.env" - $ source /home/ephios/venv/bin/activate - $ python -m ephios migrate - $ python -m ephios collectstatic --noinput - $ python -m ephios compilemessages - $ python -m ephios compilejsi18n - -Setup cron -'''''''''' - -ephios needs to have the ``run_periodic`` management command run periodically (every few minutes). -This command sends notifications and performs other tasks that need to be done regularly. -Run ``crontab -e -u ephios`` and add the following line: - -.. code-block:: bash - - */5 * * * * ENV_PATH=/home/ephios/ephios.env /home/ephios/venv/bin/python -m ephios run_periodic - -Setup gunicorn systemd service -'''''''''''''''''''''''''''''' - -To run ephios with gunicorn, create a systemd service file ``/etc/systemd/system/ephios-gunicorn.service`` -with the following content: - -.. code-block:: ini - - [Unit] - Description=ephios gunicorn daemon - After=network.target - - [Service] - Type=notify - User=ephios - Group=ephios - WorkingDirectory=/home/ephios - Environment="ENV_PATH=/home/ephios/ephios.env" - ExecStart=/home/ephios/venv/bin/gunicorn ephios.wsgi --name ephios \ - --workers 5 --max-requests 1000 --max-requests-jitter 100 --bind=127.0.0.1:8327 - Restart=on-failure - - [Install] - WantedBy=multi-user.target - -To start the service run: - -.. code-block:: console - - # systemctl daemon-reload - # systemctl enable ephios-gunicorn - # systemctl start ephios-gunicorn - - -Configure reverse proxy -''''''''''''''''''''''' - -Configure your reverse proxy to forward requests to ephios. For nginx, you can start with this: - -.. code-block:: nginx - - server { - listen 80 default_server; - listen [::]:80 ipv6only=on default_server; - server_name your.domain.org - location / { - return 301 https://$host$request_uri; - } - } - - server { - listen 443 ssl; - listen [::]:443 ipv6only=on ssl; - server_name your.domain.org; - - http2 on; - ssl_certificate /etc/letsencrypt/certificates/your.domain.org.crt; - ssl_certificate_key /etc/letsencrypt/certificates/your.domain.org.key; - - location / { - proxy_pass http://localhost:8327; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto https; - proxy_set_header Host $http_host; - proxy_redirect off; - } - - location /static/ { - alias /var/ephios/data/static/; - access_log off; - expires 1d; - add_header Cache-Control "public"; - } - } - -For apache you can build on this: - -.. code-block:: apache - - - ServerName your.domain.org - Redirect permanent / https://your.domain.org/ - - - - ServerName your.domain.org - SSLEngine on - SSLCertificateFile /etc/letsencrypt/certificates/your.domain.org.crt - SSLCertificateKeyFile /etc/letsencrypt/certificates/your.domain.org.key - - ProxyPass /static/ ! - Alias /static/ /var/ephios/data/static/ - - Require all granted - - - RequestHeader set X-Forwarded-Proto "https" - ProxyPreserveHost On - ProxyPass / http://localhost:8327/ - ProxyPassReverse / http://localhost:8327/ - - -Remember to replace all the domain names and certificate paths with your own. -Make sure to use secure SSL settings. -To obtain SSL certificates, you can use `certbot `_ with Let's Encrypt. - -Next steps -'''''''''' - -After restarting your reverse proxy you should be able to access ephios at https://your.domain.org. -You can now create your first user account by running: - -.. code-block:: console - - # sudo -u ephios -i - $ export ENV_PATH="/home/ephios/ephios.env" - $ source /home/ephios/venv/bin/activate - $ python -m ephios createsuperuser - -You should now secure your installation. Try starting with the tips below. - -To install a plugin install them via pip and restart the ephios-gunicorn service: - -.. code-block:: console - - # ENV_PATH="/home/ephios/ephios.env" sudo -u ephios /home/ephios/venv/bin/pip install ephios- - # systemctl restart ephios-gunicorn - -To update ephios create a backup of your database and files and run: - -.. code-block:: console - - # sudo -u ephios -i - $ export ENV_PATH="/home/ephios/ephios.env" - $ source /home/ephios/venv/bin/activate - $ pip install -U "ephios[redis,pgsql]" - $ python -m ephios migrate - $ python -m ephios collectstatic --noinput - $ python -m ephios compilemessages - $ python -m ephios compilejsi18n - -Then, as root, restart the gunicorn service: +.. toctree:: + :maxdepth: 1 -.. code-block:: console + manual/index + docker/index - # systemctl restart ephios-gunicorn Securing your installation -------------------------- @@ -346,6 +67,10 @@ Restart and maybe also add your domain to the `HSTS preload list `_ for ephios and install the ephios package. +Replace pgsql with mysql if you want to use mysql. + +.. code-block:: console + + # sudo -u ephios python3.11 -m venv /home/ephios/venv + # sudo -u ephios /home/ephios/venv/bin/pip install gunicorn "ephios[redis,pgsql]" + +Database +'''''''' + +Create a database user and a database for ephios. For postgres this could look like this: + +.. code-block:: console + + # sudo -u postgres createuser ephios + # sudo -u postgres createdb -O ephios ephios + +Make sure the encoding of the database is UTF-8. + +Data directory +'''''''''''''' + +ephios stores some data in the file system. Create the folders and make sure the ephios user can write to them. +The reverse proxy needs to be able to read the static files stored in ``/var/ephios/data/public/static``. + +.. code-block:: console + + # mkdir -p /var/ephios/data/ + # chown -R ephios:ephios /var/ephios + +.. _web_push_notifications: + +Config file +''''''''''' + +ephios can be configured using environment variables. They can also be read from a file. +Create a file ``/home/ephios/ephios.env`` (owned by the ephios user) with the following +content, replacing the values with your own: + +.. code-block:: + + DEBUG=False + DATA_DIR=/var/ephios/data + DATABASE_URL=psql://dbuser:dbpass@localhost:5432/ephios + ALLOWED_HOSTS="your.domain.org" + SITE_URL=https://your.domain.org + EMAIL_URL=smtp+ssl://emailuser:emailpass@smtp.domain.org:465 + DEFAULT_FROM_EMAIL=ephios@domain.org + SERVER_EMAIL=ephios@domain.org + ADMINS=Org Admin + CACHE_URL="redis://127.0.0.1:6379/1" + + +For details on the configuration options and syntax, see :ref:`configuration options `. + +To test your configuration, run: + +.. code-block:: console + + # sudo -u ephios -i + $ export ENV_PATH="/home/ephios/ephios.env" + $ source /home/ephios/venv/bin/activate + $ python -m ephios check --deploy + $ python -m ephios sendtestemail --admin + +Build ephios files +'''''''''''''''''' + +Now that the configuration is in place, we can build the static files and the translation files. + +.. code-block:: console + + # sudo -u ephios -i + $ export ENV_PATH="/home/ephios/ephios.env" + $ source /home/ephios/venv/bin/activate + $ python -m ephios migrate + $ python -m ephios build + +Setup cron +'''''''''' + +ephios needs to have the ``run_periodic`` management command run periodically (at least every five minutes). +This command sends notifications and performs other tasks that need to be done regularly. +Run ``crontab -e -u ephios`` and add the following line: + +.. code-block:: bash + + */5 * * * * ENV_PATH=/home/ephios/ephios.env /home/ephios/venv/bin/python -m ephios run_periodic + +Setup gunicorn systemd service +'''''''''''''''''''''''''''''' + +To run ephios with gunicorn, create a systemd service file ``/etc/systemd/system/ephios-gunicorn.service`` +with the following content: + +.. code-block:: ini + + [Unit] + Description=ephios gunicorn daemon + After=network.target + + [Service] + Type=notify + User=ephios + Group=ephios + WorkingDirectory=/home/ephios + Environment="ENV_PATH=/home/ephios/ephios.env" + ExecStart=/home/ephios/venv/bin/gunicorn ephios.wsgi --name ephios \ + --workers 5 --max-requests 1000 --max-requests-jitter 100 --bind=127.0.0.1:8327 + Restart=on-failure + + [Install] + WantedBy=multi-user.target + +To start the service run: + +.. code-block:: console + + # systemctl daemon-reload + # systemctl enable ephios-gunicorn + # systemctl start ephios-gunicorn + + +Configure reverse proxy +''''''''''''''''''''''' + +Configure your reverse proxy to forward requests to ephios. For nginx, you can start with this: + +.. code-block:: nginx + + server { + listen 80 default_server; + listen [::]:80 ipv6only=on default_server; + server_name your.domain.org + location / { + return 301 https://$host$request_uri; + } + } + + server { + listen 443 ssl; + listen [::]:443 ipv6only=on ssl; + server_name your.domain.org; + + http2 on; + ssl_certificate /etc/letsencrypt/certificates/your.domain.org.crt; + ssl_certificate_key /etc/letsencrypt/certificates/your.domain.org.key; + + location / { + proxy_pass http://localhost:8327; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Host $http_host; + proxy_redirect off; + } + + location /static/ { + alias /var/ephios/data/public/static/; + access_log off; + expires 1d; + add_header Cache-Control "public"; + } + } + +For apache you can build on this: + +.. code-block:: apache + + + ServerName your.domain.org + Redirect permanent / https://your.domain.org/ + + + + ServerName your.domain.org + SSLEngine on + SSLCertificateFile /etc/letsencrypt/certificates/your.domain.org.crt + SSLCertificateKeyFile /etc/letsencrypt/certificates/your.domain.org.key + + ProxyPass /static/ ! + Alias /static/ /var/ephios/data/public/static/ + + Require all granted + + + RequestHeader set X-Forwarded-Proto "https" + ProxyPreserveHost On + ProxyPass / http://localhost:8327/ + ProxyPassReverse / http://localhost:8327/ + + +Remember to replace all the domain names and certificate paths with your own. +Make sure to use secure SSL settings. +To obtain SSL certificates, you can use `certbot `_ with Let's Encrypt. + +Next steps +'''''''''' + +After restarting your reverse proxy you should be able to access ephios at https://your.domain.org. +You can now create your first user account by running: + +.. code-block:: console + + # sudo -u ephios -i + $ export ENV_PATH="/home/ephios/ephios.env" + $ source /home/ephios/venv/bin/activate + $ python -m ephios createsuperuser + +You should now secure your installation. Try starting with the tips below. + +To install a plugin install them via pip: + +.. code-block:: console + + # sudo -u ephios -i + $ export ENV_PATH="/home/ephios/ephios.env" + $ source /home/ephios/venv/bin/activate + $ pip install "ephios-" + $ python -m ephios migrate + $ python -m ephios build + +To update ephios create a backup of your database and files and run: + +.. code-block:: console + + # sudo -u ephios -i + $ export ENV_PATH="/home/ephios/ephios.env" + $ source /home/ephios/venv/bin/activate + $ pip install -U "ephios[redis,pgsql]" + $ python -m ephios migrate + $ python -m ephios build + +After installing plugins or updating, restart the gunicorn service: + +.. code-block:: console + + # systemctl restart ephios-gunicorn \ No newline at end of file diff --git a/docs/development/contributing.rst b/docs/development/contributing.rst index b0ad94d0a..fc82fd725 100644 --- a/docs/development/contributing.rst +++ b/docs/development/contributing.rst @@ -29,7 +29,7 @@ To set up a development version on your local machine, you need to execute the f #. Install dependencies with ``poetry install`` #. Create env file with ``cp .env.example .env`` #. Migrate the database with ``python manage.py migrate`` -#. Compile translations with ``python manage.py compilemessages`` and ``python manage.py compilejsi18n`` +#. Prepare files for the installation with ``python manage.py build`` #. Load data for testing with ``python manage.py devdata`` #. Start the development server with ``python manage.py runserver`` #. Open your web browser, visit ``http://localhost:8000`` and log in with the default credentials (user ``admin@localhost`` and password ``admin``) @@ -76,14 +76,12 @@ the `locale/**/.po` files for translation: Calling ``makemessages`` in the ``djangojs`` domain will find gettext calls in javascript files in the current working directory. Therefore, we need to ignore the ``data`` which contains static files from 3rd-party-packages already translated and the ``docs`` directory. Some 3rd-party-javascript comes without -a translation. To add them, do run ``makemessages`` from the ``data/static/`` directory after running +a translation. To add them, do run ``makemessages`` from the ``data/public/static/`` directory after running ``collectstatic``, but ignore all directories of 3rd-party-packages that are already translated, e.g.: .. code-block:: bash - cd data/static/ - # if django complains about not finding VAPID files when running from this bizare directory, - # prepend VAPID_PRIVATE_KEY_PATH="" to the following command - python ../../manage.py makemessages --all -d djangojs --ignore jsi18n --ignore admin --ignore CACHE --ignore recurrence --ignore select2 + cd data/public/static/ + python ../../../manage.py makemessages --all -d djangojs --ignore jsi18n --ignore admin --ignore CACHE --ignore recurrence --ignore select2 We tend to edit our .po files using weblate, but a local editor like poedit works as well. \ No newline at end of file diff --git a/ephios/core/apps.py b/ephios/core/apps.py index 49d1ffd7b..408828ea6 100644 --- a/ephios/core/apps.py +++ b/ephios/core/apps.py @@ -1,7 +1,6 @@ import logging from django.apps import AppConfig -from django.conf import settings from dynamic_preferences.registries import preference_models logger = logging.getLogger(__name__) @@ -13,9 +12,8 @@ class CoreAppConfig(AppConfig): def ready(self): from ephios.core.dynamic_preferences_registry import event_type_preference_registry + from . import checks # pylint: disable=unused-import from . import signals # pylint: disable=unused-import EventTypePreference = self.get_model("EventTypePreference") preference_models.register(EventTypePreference, event_type_preference_registry) - - logger.info("Installed plugins: %s", ", ".join(settings.PLUGINS)) diff --git a/ephios/core/checks.py b/ephios/core/checks.py new file mode 100644 index 000000000..b182beabb --- /dev/null +++ b/ephios/core/checks.py @@ -0,0 +1,52 @@ +import os + +from django.conf import settings +from django.core.checks import Error as DjangoError +from django.core.checks import Warning as DjangoWarning +from django.core.checks import register + + +@register("ephios", deploy=True) +def check_ephios_deploy_settings(app_configs, **kwargs): + # On production setups, the data directory should + # not be inside the base directory (ephios package directory). + errors = [] + + bad_directories = [] + for name, directory in settings.DIRECTORIES.items(): + if os.path.commonpath([settings.BASE_DIR, directory]) == settings.BASE_DIR: + bad_directories.append((name, directory)) + if bad_directories: + errors.append( + DjangoWarning( + "ephios data is stored near the ephios package directory.", + hint=f"You probably don't want to have the {', '.join(name for name, _ in bad_directories)} " + f"be inside or next to the ephios python package at " + f"{settings.BASE_DIR}! " + f"Configure ephios to use another location.", + id="ephios.W001", + ) + ) + + # Check that VAPID keys were used to configure webpush. + if not settings.WEBPUSH_SETTINGS: + errors.append( + DjangoWarning( + "WEBPUSH_SETTINGS are not configured.", + hint="You need to generate a vapid key using ./manage.py generate_vapid_key in order to " + "use push notifications.", + id="ephios.W002", + ) + ) + + if not os.access(settings.MEDIA_ROOT, os.W_OK): + errors.append( + DjangoError( + "Media root not writable by application server.", + hint=f"You need to make sure that the application server can write to the media root at " + f"{settings.MEDIA_ROOT}.", + id="ephios.E001", + ) + ) + + return errors diff --git a/ephios/core/dynamic_preferences_registry.py b/ephios/core/dynamic_preferences_registry.py index c9a661faf..a8c5c53ef 100644 --- a/ephios/core/dynamic_preferences_registry.py +++ b/ephios/core/dynamic_preferences_registry.py @@ -1,5 +1,10 @@ +from datetime import datetime, timedelta + +from django.conf import settings from django.contrib.auth.models import Group +from django.utils import timezone from django.utils.safestring import mark_safe +from django.utils.timezone import make_aware from django.utils.translation import gettext_lazy as _ from django_select2.forms import Select2MultipleWidget from dynamic_preferences.preferences import Section @@ -8,6 +13,7 @@ global_preferences_registry, ) from dynamic_preferences.types import ( + DateTimePreference, ModelMultipleChoicePreference, MultipleChoicePreference, StringPreference, @@ -31,6 +37,7 @@ class EventTypeRegistry(PerInstancePreferenceRegistry): notifications_user_section = Section("notifications") responsible_notifications_user_section = Section("responsible_notifications") general_global_section = Section("general") +internal_section = Section("internal") # for settings/stats that should not be exposed to users @global_preferences_registry.register @@ -62,6 +69,38 @@ def get_choices(): ] +@global_preferences_registry.register +class LastRunPeriodicCall(DateTimePreference): + NONE_VALUE = make_aware(datetime(1970, 1, 1)) + name = "last_run_periodic_call" + verbose_name = _("Last run periodic call") + section = internal_section + required = False + + def get_default(self): + return self.NONE_VALUE + + @classmethod + def get_last_call(cls): + preferences = global_preferences_registry.manager() + last_call = preferences[f"{cls.section.name}__{cls.name}"] + if last_call == cls.NONE_VALUE: + return None + return last_call + + @classmethod + def is_stuck(cls): + last_call = cls.get_last_call() + return last_call is None or ( + (timezone.now() - last_call) > timedelta(seconds=settings.RUN_PERIODIC_MAX_INTERVAL) + ) + + @classmethod + def set_last_call(cls, value): + preferences = global_preferences_registry.manager() + preferences[f"{cls.section.name}__{cls.name}"] = value + + @user_preferences_registry.register class NotificationPreference(JSONPreference): name = "notifications" diff --git a/ephios/extra/management/__init__.py b/ephios/core/management/__init__.py similarity index 100% rename from ephios/extra/management/__init__.py rename to ephios/core/management/__init__.py diff --git a/ephios/extra/management/commands/__init__.py b/ephios/core/management/commands/__init__.py similarity index 100% rename from ephios/extra/management/commands/__init__.py rename to ephios/core/management/commands/__init__.py diff --git a/ephios/core/management/commands/build.py b/ephios/core/management/commands/build.py new file mode 100644 index 000000000..63e92cb64 --- /dev/null +++ b/ephios/core/management/commands/build.py @@ -0,0 +1,11 @@ +from django.core.management import BaseCommand, call_command + + +class Command(BaseCommand): + help = "(Re)build static files, language files, directories and VAPID key" + + def handle(self, *args, **options): + call_command("generate_vapid_key", verbosity=1) + call_command("collectstatic", verbosity=1, interactive=False) + call_command("compilemessages", verbosity=1) + call_command("compilejsi18n", verbosity=1) diff --git a/ephios/extra/management/commands/devdata.py b/ephios/core/management/commands/devdata.py similarity index 100% rename from ephios/extra/management/commands/devdata.py rename to ephios/core/management/commands/devdata.py diff --git a/ephios/core/management/commands/generate_vapid_key.py b/ephios/core/management/commands/generate_vapid_key.py new file mode 100644 index 000000000..211e93455 --- /dev/null +++ b/ephios/core/management/commands/generate_vapid_key.py @@ -0,0 +1,20 @@ +import os.path + +from django.conf import settings +from django.core.management import BaseCommand +from py_vapid import Vapid + + +class Command(BaseCommand): + help = "Generate VAPID key at VAPID_PRIVATE_KEY_PATH if it does not exist" + + def handle(self, *args, **options): + if not os.path.exists(settings.VAPID_PRIVATE_KEY_PATH): + print("Generating VAPID key") + vapid = Vapid() + vapid.generate_keys() + vapid.save_key(settings.VAPID_PRIVATE_KEY_PATH) + vapid.save_public_key(settings.VAPID_PRIVATE_KEY_PATH + ".pub") + print(f"Saved to {settings.VAPID_PRIVATE_KEY_PATH}") + else: + print(f"VAPID key already exists at {settings.VAPID_PRIVATE_KEY_PATH}") diff --git a/ephios/extra/management/commands/run_periodic.py b/ephios/core/management/commands/run_periodic.py similarity index 60% rename from ephios/extra/management/commands/run_periodic.py rename to ephios/core/management/commands/run_periodic.py index 23753ce73..7d33375ed 100644 --- a/ephios/extra/management/commands/run_periodic.py +++ b/ephios/core/management/commands/run_periodic.py @@ -1,8 +1,15 @@ +import logging + from django.core.management import BaseCommand from ephios.core.signals import periodic_signal +logger = logging.getLogger(__name__) + class Command(BaseCommand): + help = "Run periodic tasks" + def handle(self, *args, **options): + logger.info("Running periodic tasks") periodic_signal.send(self) diff --git a/ephios/extra/management/commands/send_notifications.py b/ephios/core/management/commands/send_notifications.py similarity index 73% rename from ephios/extra/management/commands/send_notifications.py rename to ephios/core/management/commands/send_notifications.py index cc1bca385..cd3c2899b 100644 --- a/ephios/extra/management/commands/send_notifications.py +++ b/ephios/core/management/commands/send_notifications.py @@ -4,5 +4,7 @@ class Command(BaseCommand): + help = "Send all notifications (for testing, use run_periodic in production)" + def handle(self, *args, **options): send_all_notifications() diff --git a/ephios/core/plugins.py b/ephios/core/plugins.py index 3140b8046..d070e99fb 100644 --- a/ephios/core/plugins.py +++ b/ephios/core/plugins.py @@ -7,8 +7,6 @@ logger = logging.getLogger(__name__) -# The plugin mechanics are heavily inspired by pretix (then licenced under Apache 2.0) - Check it out! - def get_all_plugins(): """ diff --git a/ephios/core/services/health/healthchecks.py b/ephios/core/services/health/healthchecks.py new file mode 100644 index 000000000..0766e9924 --- /dev/null +++ b/ephios/core/services/health/healthchecks.py @@ -0,0 +1,199 @@ +import os + +from django.conf import settings +from django.contrib.auth.models import Permission +from django.contrib.humanize.templatetags.humanize import naturaltime +from django.dispatch import receiver +from django.template.defaultfilters import floatformat +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +from ephios.core.dynamic_preferences_registry import LastRunPeriodicCall +from ephios.core.signals import register_healthchecks + +# health checks are meant to monitor the health of the application while it is running +# in contrast there are django checks which are meant to check the configuration of the application + + +def run_healthchecks(): + for _, healthchecks in register_healthchecks.send(None): + for HealthCheck in healthchecks: + check = HealthCheck() + status, message = check.check() + yield check, status, message + + +class HealthCheckStatus: + OK = "ok" + WARNING = "warning" + ERROR = "error" + + +class AbstractHealthCheck: + @property + def slug(self): + """ + Return a unique slug for this health check. + """ + raise NotImplementedError + + @property + def name(self): + """ + Return a short name of this health check. + """ + raise NotImplementedError + + @property + def description(self): + """ + Return a short description of this health check. + """ + raise NotImplementedError + + @property + def documentation_link(self): + """ + Return a link to the documentation of this health check. + """ + return None + + def check(self): + """ + Return a tuple of (status, message) where status is one of HealthCheckStatus + """ + raise NotImplementedError + + +class DBHealthCheck(AbstractHealthCheck): + slug = "db" + name = _("Database") + description = _("The database is the central storage for all data.") + documentation_link = "https://docs.djangoproject.com/en/stable/ref/databases/" + + def check(self): + from django.db import connection + + try: + connection.cursor() + Permission.objects.exists() + except Exception as e: + return HealthCheckStatus.ERROR, str(e) + + if settings.DATABASES["default"]["ENGINE"] == "django.db.backends.sqlite3": + return HealthCheckStatus.WARNING, _( + "Using SQLite, which should not be used in production." + ) + + return HealthCheckStatus.OK, _("Database connection established.") + + +class CacheHealthCheck(AbstractHealthCheck): + slug = "cache" + name = _("Cache") + description = _("The cache is used to store temporary data.") + documentation_link = "https://docs.djangoproject.com/en/stable/topics/cache/" + + def check(self): + from django.core import cache + + try: + cache.cache.set("_healthcheck", "1") + if not cache.cache.get("_healthcheck") == "1": + raise Exception("Cache not available") + except Exception as e: + return HealthCheckStatus.ERROR, str(e) + + if ( + settings.CACHES.get("default", {}).get("BACKEND") + == "django.core.cache.backends.locmem.LocMemCache" + ): + return HealthCheckStatus.WARNING, _( + "Using LocMemCache, which should not be used in production." + ) + + return HealthCheckStatus.OK, _("Cache connection established.") + + +class CronJobHealthCheck(AbstractHealthCheck): + slug = "cronjob" + name = _("Cronjob") + description = _( + "A cron job must regularly call ephios to do recurring tasks like sending notifications." + ) + documentation_link = ( + "https://docs.ephios.de/en/stable/admin/deployment/manual/index.html#setup-cron" + ) + + def check(self): + last_call = LastRunPeriodicCall.get_last_call() + if LastRunPeriodicCall.is_stuck(): + if last_call: + return ( + HealthCheckStatus.WARNING, + mark_safe( + _("Cronjob stuck, last run {last_call}.").format( + last_call=naturaltime(last_call), + ) + ), + ) + else: + return ( + HealthCheckStatus.ERROR, + mark_safe(_("Cronjob stuck, no last run.")), + ) + else: + return ( + HealthCheckStatus.OK, + mark_safe(_("Last run {last_call}.").format(last_call=naturaltime(last_call))), + ) + + +class DiskSpaceHealthCheck(AbstractHealthCheck): + slug = "disk_space" + name = _("Disk space") + description = _("Disk space needs to be available to store data.") + documentation_link = ( + "https://docs.ephios.de/en/stable/admin/deployment/manual/index.html#data-directory" + ) + + def check(self): + # if under 100 MB are available, we consider this an error + # if under 1 GB are available, we consider this a warning + # otherwise, we consider this ok + disk_usage = os.statvfs(settings.MEDIA_ROOT) + free_bytes = disk_usage.f_bavail * disk_usage.f_frsize + MEGA = 1024 * 1024 + if free_bytes < 100 * MEGA: + return ( + HealthCheckStatus.ERROR, + mark_safe( + _( + "Less than 100 MB of disk space available. " + "Please free up some disk space." + ) + ), + ) + if free_bytes < 1024 * MEGA: + return ( + HealthCheckStatus.WARNING, + mark_safe( + _("Less than 1 GB of disk space available. Please free up some disk space.") + ), + ) + return ( + HealthCheckStatus.OK, + mark_safe( + _("{disk_space} of disk space available.").format( + disk_space=f"{floatformat(free_bytes / MEGA / 1024,1)} GB" + ) + ), + ) + + +@receiver(register_healthchecks, dispatch_uid="ephios.core.healthchecks.register_core_healthchecks") +def register_core_healthchecks(sender, **kwargs): + yield DBHealthCheck + yield CacheHealthCheck + yield CronJobHealthCheck + yield DiskSpaceHealthCheck diff --git a/ephios/core/signals.py b/ephios/core/signals.py index bfe333cbd..55563482e 100644 --- a/ephios/core/signals.py +++ b/ephios/core/signals.py @@ -1,4 +1,5 @@ from django.dispatch import receiver +from django.utils import timezone from ephios.core.plugins import PluginSignal from ephios.core.services.notifications.backends import send_all_notifications @@ -94,6 +95,12 @@ Receivers should return a list of subclasses of ``ephios.core.notifications.backends.AbstractNotificationBackend`` """ +register_healthchecks = PluginSignal() +""" +This signal is sent out to get all health checks that can be run to monitor the health of the application. +Receivers should return a list of subclasses of ``ephios.core.services.health.AbstractHealthCheck`` +""" + periodic_signal = PluginSignal() """ This signal is called periodically, at least every 15 minutes. @@ -174,6 +181,13 @@ def send_notifications(sender, **kwargs): send_all_notifications() +@receiver(periodic_signal, dispatch_uid="ephios.core.signals.update_last_run_periodic_call") +def update_last_run_periodic_call(sender, **kwargs): + from ephios.core.dynamic_preferences_registry import LastRunPeriodicCall + + LastRunPeriodicCall.set_last_call(timezone.now()) + + periodic_signal.connect( send_participation_finished, dispatch_uid="ephios.core.signals.send_participation_finished" ) diff --git a/ephios/core/templates/core/settings/settings_instance.html b/ephios/core/templates/core/settings/settings_instance.html index aa540c464..6adfce167 100644 --- a/ephios/core/templates/core/settings/settings_instance.html +++ b/ephios/core/templates/core/settings/settings_instance.html @@ -1,11 +1,57 @@ {% extends "core/settings/settings_base.html" %} +{% load humanize %} {% load crispy_forms_filters %} {% load i18n %} {% block settings_content %} -
+ {% csrf_token %} {{ form|crispy }}
+ + {% if healthchecks %} + +

+ {% translate "System health" %} +

+
+ {% for check, status, message in healthchecks %} +
+
+
+
+ {{ check.name }} + {% if status == "error" %} + + + {% elif status == "warning" %} + + + {% elif status == "ok" %} + + + {% endif %} +
+

+ {{ check.description }} + {% if check.documentation_link %} + + + + + {% endif %} +

+

+ {{ message }} +

+
+
+
+ {% endfor %} +
+ {% endif %} + {% endblock %} diff --git a/ephios/core/urls.py b/ephios/core/urls.py index 27ebc5d0d..a1bbf8830 100644 --- a/ephios/core/urls.py +++ b/ephios/core/urls.py @@ -39,6 +39,7 @@ EventTypeListView, EventTypeUpdateView, ) +from ephios.core.views.healthcheck import HealthCheckView from ephios.core.views.log import LogView from ephios.core.views.pwa import OfflineView, PWAManifestView, ServiceWorkerView from ephios.core.views.settings import ( @@ -68,6 +69,10 @@ app_name = "core" urlpatterns = [ path("", HomeView.as_view(), name="home"), + path("manifest.json", PWAManifestView.as_view(), name="pwa_manifest"), + path("serviceworker.js", ServiceWorkerView.as_view(), name="pwa_serviceworker"), + path("offline/", OfflineView.as_view(), name="pwa_offline"), + path("healthcheck/", HealthCheckView.as_view(), name="healthcheck"), path("events/", EventListView.as_view(), name="event_list"), path( "events//edit/", @@ -225,9 +230,6 @@ name="consequence_edit", ), path("log/", LogView.as_view(), name="log"), - path("manifest.json", PWAManifestView.as_view(), name="pwa_manifest"), - path("serviceworker.js", ServiceWorkerView.as_view(), name="pwa_serviceworker"), - path("offline/", OfflineView.as_view(), name="pwa_offline"), path("workinghours/own/", OwnWorkingHourView.as_view(), name="workinghours_own"), path( "workinghours/own/request/", diff --git a/ephios/core/views/healthcheck.py b/ephios/core/views/healthcheck.py new file mode 100644 index 000000000..358b6c285 --- /dev/null +++ b/ephios/core/views/healthcheck.py @@ -0,0 +1,26 @@ +from django.http import HttpResponse +from django.views import View + +from ephios.core.services.health.healthchecks import HealthCheckStatus, run_healthchecks + + +class HealthCheckView(View): + def get(self, request, *args, **kwargs): + okays = [] + not_okays = [] + + for check, status, message in run_healthchecks(): + text = f"{check.name}: {message}" + if status in {HealthCheckStatus.OK, HealthCheckStatus.WARNING}: + okays.append(text) + else: + not_okays.append(text) + + status = 200 + message = "" + if not_okays: + status = 503 + message += "NOT OK

" + "
".join(not_okays) + "

" + if okays: + message += "OK

" + "
".join(okays) + "

" + return HttpResponse(message, status=status) diff --git a/ephios/core/views/settings.py b/ephios/core/views/settings.py index b9aac4811..92bc336be 100644 --- a/ephios/core/views/settings.py +++ b/ephios/core/views/settings.py @@ -7,6 +7,7 @@ from dynamic_preferences.forms import global_preference_form_builder from ephios.core.forms.users import UserNotificationPreferenceForm +from ephios.core.services.health.healthchecks import run_healthchecks from ephios.core.signals import management_settings_sections from ephios.extra.mixins import StaffRequiredMixin @@ -55,6 +56,11 @@ def form_valid(self, form): def get_success_url(self): return reverse("core:settings_instance") + def get_context_data(self, **kwargs): + if self.request.user.is_superuser: + kwargs["healthchecks"] = list(run_healthchecks()) + return super().get_context_data(**kwargs) + class PersonalDataSettingsView(LoginRequiredMixin, TemplateView): template_name = "core/settings/settings_personal_data.html" diff --git a/ephios/extra/secrets.py b/ephios/extra/secrets.py new file mode 100644 index 000000000..c93d37efb --- /dev/null +++ b/ephios/extra/secrets.py @@ -0,0 +1,21 @@ +import os +import string + +from django.utils.crypto import get_random_string + + +def django_secret_from_file(path: str): + if os.path.exists(path): + with open(path, "r", encoding="utf8") as f: + return f.read().strip() + chars = string.ascii_letters + string.digits + string.punctuation + secret = get_random_string(50, chars) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf8") as f: + os.chmod(path, 0o600) + try: + os.chown(path, os.getuid(), os.getgid()) + except AttributeError: + pass # os.chown is not available on Windows + f.write(secret) + return secret diff --git a/ephios/settings.py b/ephios/settings.py index 210c21fd4..218f2f901 100644 --- a/ephios/settings.py +++ b/ephios/settings.py @@ -11,13 +11,17 @@ from django.utils.translation import gettext_lazy from py_vapid import Vapid, b64urlencode +from ephios.extra.secrets import django_secret_from_file + try: import importlib_metadata # importlib is broken on python3.8, using backport except ImportError: import importlib.metadata as importlib_metadata +# BASE_DIR is the directory containing the ephios package BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + env = environ.Env() # for syntax see https://django-environ.readthedocs.io/en/latest/ # read env file from ENV_PATH or fall back to a .env file in the project root @@ -25,14 +29,35 @@ print(f"Loading ephios environment from {Path(env_path).absolute()}") environ.Env.read_env(env_file=env_path) -SECRET_KEY = env.str("SECRET_KEY") DEBUG = env.bool("DEBUG") -ALLOWED_HOSTS = env.list("ALLOWED_HOSTS") -data_dir = os.path.join(BASE_DIR, "data") -if DEBUG and not os.path.exists(data_dir): - # create data dir for development - os.mkdir(data_dir) +DATA_DIR = env.str("DATA_DIR", os.path.join(BASE_DIR, "data")) +PUBLIC_DIR = env.str("PUBLIC_DIR", os.path.join(DATA_DIR, "public")) +STATIC_ROOT = env.str("STATIC_ROOT", os.path.join(PUBLIC_DIR, "static")) + +PRIVATE_DIR = env.str("PRIVATE_DIR", os.path.join(DATA_DIR, "private")) +LOG_DIR = env.str("LOG_DIR", os.path.join(PRIVATE_DIR, "logs")) +MEDIA_ROOT = env.str("MEDIA_ROOT", os.path.join(PRIVATE_DIR, "media")) + +DIRECTORIES = { + "DATA_DIR": DATA_DIR, + "PUBLIC_DIR": PUBLIC_DIR, + "STATIC_ROOT": STATIC_ROOT, + "PRIVATE_DIR": PRIVATE_DIR, + "LOG_DIR": LOG_DIR, + "MEDIA_ROOT": MEDIA_ROOT, +} +for directory in DIRECTORIES.values(): + os.makedirs(directory, exist_ok=True) + +if "SECRET_KEY" in env: + SECRET_KEY = env.str("SECRET_KEY") +else: + SECRET_FILE = os.path.join(PRIVATE_DIR, ".secret") + SECRET_KEY = django_secret_from_file(SECRET_FILE) + + +ALLOWED_HOSTS = env.list("ALLOWED_HOSTS") try: EPHIOS_VERSION = importlib_metadata.version("ephios") @@ -40,20 +65,6 @@ # ephios is not installed as a package (e.g. development setup) EPHIOS_VERSION = "dev" -if not DEBUG: - SESSION_COOKIE_SECURE = True - CSRF_COOKIE_SECURE = True - X_FRAME_OPTIONS = "DENY" - SECURE_CONTENT_TYPE_NOSNIFF = True - SECURE_SSL_REDIRECT = True - SECURE_REFERRER_POLICY = "same-origin" - # 1 day by default, change to 1 year in production (see deployment docs) - SECURE_HSTS_SECONDS = env.int("SECURE_HSTS_SECONDS", default=3600 * 24) - SECURE_HSTS_INCLUDE_SUBDOMAINS = env.bool("SECURE_HSTS_INCLUDE_SUBDOMAINS", default=False) - SECURE_HSTS_PRELOAD = env.bool("SECURE_HSTS_PRELOAD", default=False) - - CONN_MAX_AGE = env.int("CONN_MAX_AGE", default=0) - INSTALLED_APPS = [ # we need to import our own modules before everything else to allow template # customizing e.g. for django-oauth-toolkit @@ -207,10 +218,6 @@ STATIC_URL = env.str("STATIC_URL", default="/static/") -STATIC_ROOT = env.str("STATIC_ROOT") -if not os.path.isabs(STATIC_ROOT): - STATIC_ROOT = os.path.join(BASE_DIR, STATIC_ROOT) - STATICFILES_DIRS = (os.path.join(BASE_DIR, "ephios/static"),) STATICFILES_FINDERS = ( "django.contrib.staticfiles.finders.FileSystemFinder", @@ -229,11 +236,6 @@ ADMINS = getaddresses([env("ADMINS")]) # logging -LOGGING_FILE = env.str("LOGGING_FILE", default=None) -use_file_logging = not DEBUG and LOGGING_FILE is not None -if use_file_logging: - Path(LOGGING_FILE).parent.mkdir(parents=True, exist_ok=True) - LOGGING = { "version": 1, "disable_existing_loggers": False, @@ -263,20 +265,16 @@ "file": { "level": "DEBUG", "formatter": "default", - "filters": ["require_debug_false"], + "filters": [], **( { "class": "logging.handlers.TimedRotatingFileHandler", - "filename": LOGGING_FILE, + "filename": os.path.join(LOG_DIR, "ephios.log"), "when": "midnight", "backupCount": env.int("LOGGING_BACKUP_DAYS", default=14), "atTime": datetime.time(4), "encoding": "utf-8", } - if use_file_logging - else { - "class": "logging.NullHandler", - } ), }, }, @@ -303,6 +301,14 @@ }, } + +def GET_SITE_URL(): + site_url = env.str("SITE_URL") + if site_url.endswith("/"): + site_url = site_url[:-1] + return site_url + + # Guardian configuration ANONYMOUS_USER_NAME = None GUARDIAN_MONKEY_PATCH = False @@ -321,7 +327,7 @@ INSTALLED_APPS.append("django_extensions") INSTALLED_APPS.append("debug_toolbar") MIDDLEWARE.insert(0, "debug_toolbar.middleware.DebugToolbarMiddleware") - INTERNAL_IPS = env.str("INTERNAL_IPS") + INTERNAL_IPS = env.list("INTERNAL_IPS", default=["127.0.0.1"]) # django-csp # Bootstrap requires embedded SVG files loaded via a data URI. This is not ideal, but will only be fixed in @@ -374,8 +380,12 @@ ] # django-webpush -if vapid_private_key_path := env.str("VAPID_PRIVATE_KEY_PATH", None): - vp = Vapid().from_file(vapid_private_key_path) +VAPID_PRIVATE_KEY_PATH = env.str( + "VAPID_PRIVATE_KEY_PATH", os.path.join(PRIVATE_DIR, "vapid_key.pem") +) +WEBPUSH_SETTINGS = {} +if os.path.exists(VAPID_PRIVATE_KEY_PATH): + vp = Vapid().from_file(VAPID_PRIVATE_KEY_PATH) WEBPUSH_SETTINGS = { "VAPID_PUBLIC_KEY": b64urlencode( vp.public_key.public_bytes( @@ -387,15 +397,13 @@ } -def GET_SITE_URL(): - site_url = env.str("SITE_URL") - if site_url.endswith("/"): - site_url = site_url[:-1] - return site_url +# Health check +# interval for calls to the run_periodic_tasks management command over which the cronjob is considered to be broken +RUN_PERIODIC_MAX_INTERVAL = 60 * 5 + 30 # 5 minutes + 30 seconds +# django-rest-framework DEFAULT_LISTVIEW_PAGINATION = 100 - REST_FRAMEWORK = { "DEFAULT_PERMISSION_CLASSES": ["rest_framework.permissions.DjangoObjectPermissions"], "DEFAULT_VERSIONING_CLASS": "rest_framework.versioning.NamespaceVersioning", @@ -431,6 +439,25 @@ def GET_SITE_URL(): "REFRESH_TOKEN_EXPIRE_SECONDS": timedelta(days=90), } +# SECURITY SETTINGS +if not DEBUG: + SESSION_COOKIE_SECURE = True + CSRF_COOKIE_SECURE = True + X_FRAME_OPTIONS = "DENY" + SECURE_CONTENT_TYPE_NOSNIFF = True + SECURE_SSL_REDIRECT = True + SECURE_REFERRER_POLICY = "same-origin" + # 1 day by default, change to 1 year in production (see deployment docs) + SECURE_HSTS_SECONDS = env.int("SECURE_HSTS_SECONDS", default=3600 * 24) + SECURE_HSTS_INCLUDE_SUBDOMAINS = env.bool("SECURE_HSTS_INCLUDE_SUBDOMAINS", default=False) + SECURE_HSTS_PRELOAD = env.bool("SECURE_HSTS_PRELOAD", default=False) + CONN_MAX_AGE = env.int("CONN_MAX_AGE", default=0) + +if env.bool("TRUST_X_FORWARDED_PROTO", default=False): + SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") + + +# OIDC SETTINGS if ENABLE_OIDC_CLIENT := env.bool("ENABLE_OIDC_CLIENT", False): INSTALLED_APPS.append("mozilla_django_oidc") AUTHENTICATION_BACKENDS.append("ephios.extra.auth.EphiosOIDCAB") diff --git a/poetry.lock b/poetry.lock index 9022d834f..2f2dc73f2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2491,19 +2491,20 @@ files = [ [[package]] name = "urllib3" -version = "1.26.16" +version = "2.0.4" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +python-versions = ">=3.7" files = [ - {file = "urllib3-1.26.16-py2.py3-none-any.whl", hash = "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f"}, - {file = "urllib3-1.26.16.tar.gz", hash = "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14"}, + {file = "urllib3-2.0.4-py3-none-any.whl", hash = "sha256:de7df1803967d2c2a98e4b11bb7d6bd9210474c46e8a0401514e3a42a75ebde4"}, + {file = "urllib3-2.0.4.tar.gz", hash = "sha256:8d22f86aae8ef5e410d4f539fde9ce6b2113a001bb4d189e0aed70642d602b11"}, ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] -socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] [[package]] name = "virtualenv" @@ -2693,4 +2694,4 @@ redis = ["redis"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "19fe437bbd665faddf5161182386e503e31081c4d66f7dbb170644a51cc4a710" +content-hash = "6254c638d6bcd04a3b1501c7a5bd7d68cacd98dd762fd775434c192464805bef" diff --git a/pyproject.toml b/pyproject.toml index e33364903..aacd42f2e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ mysqlclient = {version = "^2.1.1", optional = true} redis = {extras = ["hiredis"], version = "^5.0.0", optional = true} django-compressor = "^4.3" django-statici18n = "^2.3.1" -django-dynamic-preferences = "^1.13.0" +django-dynamic-preferences = "^1.15.0" django-crispy-forms = "^2.0" django-webpush = "^0.3.5" importlib-metadata = { version = ">=5.2.0", python = "<3.9" } @@ -42,7 +42,7 @@ uritemplate = "^4.1.1" django-recurrence = "^1.11.1" django-oauth-toolkit = "^2.3.0" mozilla-django-oidc = "^3.0.0" -urllib3 = "^1.26.0,<2.0.0" # pinned because of poetry issues with urllib3 2.0.0 +urllib3 = "^2.0.4" pyyaml = "^6.0.1" lxml = "^4.9.3" beautifulsoup4 = "^4.12.2" diff --git a/tests/core/test_views_settings.py b/tests/core/test_views_settings.py index 75b1b4c90..dff60bcfe 100644 --- a/tests/core/test_views_settings.py +++ b/tests/core/test_views_settings.py @@ -10,5 +10,11 @@ def test_settings_calendar(django_app, volunteer): response = django_app.get(reverse("core:settings_calendar"), user=volunteer) calendar_url = response.html.find("input", id="calendar-url")["value"] assert calendar_url + response = django_app.get(calendar_url, user=volunteer) assert response + + +def test_settings_instance(django_app, superuser): + response = django_app.get(reverse("core:settings_instance"), user=superuser) + assert "System health" in response.html.text