Fix six issues preventing a fresh production install

calliope_app/api/models/engage.py

@@ -5,6 +5,8 @@
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.core.mail import send_mail
+from django.db.models.signals import post_save
+from django.dispatch import receiver
 from django.template.loader import render_to_string
 from django.urls import reverse
 from django.utils.html import mark_safe
@@ -148,3 +150,27 @@ class Meta:
 
     def __str__(self):
         return f"{self.year}, {self.month}, {self.total}"
+
+
+@receiver(post_save, sender=User)
+def ensure_user_profile(sender, instance, created, **kwargs):
+    """Give every User a User_Profile, however the User was created.
+
+    Only the registration view used to create one. An account made any other
+    way -- `createsuperuser`, the Django admin, a management script -- had no
+    profile, and templates that read `user.user_profile.timezone` then rendered
+    an empty string, because Django's template engine silently swallows
+    ObjectDoesNotExist. `{{ value|timezone:"" }}` raises
+
+        ValueError: ZoneInfo keys must be normalized relative paths
+
+    so the model page returned a 500 for that user and worked for everyone
+    else. `createsuperuser` is the documented way to make an admin account, so
+    following the setup guide produced an account that could not browse models.
+
+    The timezone default lives on the field, so a profile created here is
+    complete; activation_uuid likewise.
+    """
+    if not created:
+        return
+    User_Profile.objects.get_or_create(user=instance)

calliope_app/api/views/outputs.py

@@ -122,7 +122,7 @@ def build(request):
         try:
             compute_environment = ComputeEnvironment.objects.get(name=run_env)
         except ComputeEnvironment.DoesNotExist:
-            compute_environment = ComputeEnvironment.objects.filter(is_default=True).first(0)
+            compute_environment = ComputeEnvironment.objects.filter(is_default=True).first()
 
         timestamp = datetime.now().strftime("%Y-%m-%d %H%M%S").lower().replace(" ", "-")
         if not years:
@@ -185,6 +185,11 @@ def build(request):
                 run_parameter= Run_Parameter.objects.get(pk=int(id))
                 run.run_options.append({'root':run_parameter.root,'name':run_parameter.name,'value':parameters[id]})
 
+            # Persist before dispatching. build_model re-reads this Run from
+            # the database, so anything still only in memory is invisible to it
+            # -- and the worker can begin before the save below lands.
+            run.save()
+
             # Celery task
             async_result = build_model.apply_async(
                 kwargs={

calliope_app/calliope_app/settings/prod.py

@@ -51,9 +51,17 @@
 # https://docs.djangoproject.com/en/dev/ref/settings/#secure-ssl-redirect
 SECURE_SSL_REDIRECT = env.bool("DJANGO_SECURE_SSL_REDIRECT", default=True)
 # https://docs.djangoproject.com/en/dev/ref/settings/#session-cookie-secure
-SESSION_COOKIE_SECURE = True
+# Secure cookies are only sent over HTTPS. That is correct for a real
+# deployment, but makes login impossible on a plain-HTTP instance -- the
+# browser withholds the CSRF cookie and every form submission 403s. Both
+# default to True so existing deployments are unchanged; only an explicitly
+# insecure test instance turns them off.
+SESSION_COOKIE_SECURE = env.bool("DJANGO_SESSION_COOKIE_SECURE", default=True)
 # https://docs.djangoproject.com/en/dev/ref/settings/#csrf-cookie-secure
-CSRF_COOKIE_SECURE = True
+CSRF_COOKIE_SECURE = env.bool("DJANGO_CSRF_COOKIE_SECURE", default=True)
+# Origins Django will accept form posts from, e.g. https://engage.example.org
+# or http://1.2.3.4:8000 when running without TLS.
+CSRF_TRUSTED_ORIGINS = env.list("DJANGO_CSRF_TRUSTED_ORIGINS", default=[])
 # https://docs.djangoproject.com/en/dev/ref/settings/#secure-hsts-include-subdomains
 SECURE_HSTS_INCLUDE_SUBDOMAINS = env.bool("DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS", default=True)
 # https://docs.djangoproject.com/en/dev/ref/settings/#secure-hsts-preload
@@ -75,6 +83,24 @@
     )
 ]
 
+# STATIC
+# ------------------------------------------------------------------------------
+# DEBUG is False here, so Django no longer serves STATIC_ROOT itself. WhiteNoise
+# serves it from inside the gunicorn process, which keeps a single-container
+# deployment from needing a separate web server in front of it.
+MIDDLEWARE.insert(  # noqa F405
+    MIDDLEWARE.index("django.middleware.security.SecurityMiddleware") + 1,  # noqa F405
+    "whitenoise.middleware.WhiteNoiseMiddleware"
+)
+STORAGES = {
+    "default": {
+        "BACKEND": "django.core.files.storage.FileSystemStorage"
+    },
+    "staticfiles": {
+        "BACKEND": "whitenoise.storage.CompressedStaticFilesStorage"
+    }
+}
+
 # Gunicorn
 # ------------------------------------------------------------------------------
 INSTALLED_APPS += ["gunicorn"]  # noqa F405
@@ -165,9 +191,14 @@
 CELERY_TIMEZONE = TIME_ZONE if USE_TZ else None
 CELERY_TRACK_STARTED = True
 CELERYD_CONCURRENCY = 2
+# TLS to the broker. Managed brokers (ElastiCache in transit-encryption mode)
+# require it; a plain redis container on the same host cannot speak it at all,
+# and leaving this on against one fails every worker connection. Defaults to on
+# so existing deployments are unchanged.
+_CELERY_USE_SSL = env.bool("CELERY_BROKER_USE_SSL", default=True)
 CELERY_BROKER_USE_SSL = {
     "ssl_cert_reqs": ssl.CERT_REQUIRED
-}
+} if _CELERY_USE_SSL else None
 CELERY_REDIS_BACKEND_USE_SSL = {
     "ssl_cert_reqs": ssl.CERT_REQUIRED
-}
+} if _CELERY_USE_SSL else None

calliope_app/compose/Dockerfile

@@ -26,6 +26,10 @@ RUN apt-get update -y --fix-missing \
     git \
     && rm -rf /var/lib/apt/lists/*
 
+# Django's production logging config (settings/prod.py) writes to this path.
+# Nothing creates it at runtime, so the directory must exist in the image.
+RUN mkdir -p /opt/python/log
+
 # install python packages
 WORKDIR /www
 COPY requirements.txt requirements-dev.txt /www/

calliope_app/compose/run-calliope-app.sh

@@ -1,15 +1,9 @@
 #!/usr/bin/env bash
 
-python3 manage.py loaddata --app template \
-      admin_template_type.json \
-      admin_template_type_variables.json \
-      admin_template_type_techs.json \
-      admin_template_type_locs.json \
-      admin_template_type_loc_techs.json \
-      admin_template_type_loc_tech_params.json \
-      admin_template_type_tech_params.json \
-      admin_template_type_carriers.json
-
 python3 manage.py migrate
 
+# Seeds reference data only into an empty database -- see the script for why
+# loading these fixtures unconditionally is dangerous.
+compose/seed-reference-data.sh
+
 python3 manage.py runserver 0.0.0.0:8000

calliope_app/compose/seed-reference-data.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+#
+# Seed Engage's reference data -- abstract technologies, parameters, run
+# parameters and model templates -- but ONLY into a database that doesn't
+# already have it.
+#
+# WHY THE CHECK MATTERS
+#
+# These fixtures carry hardcoded primary keys: admin_parameter.json is pk 1
+# through 154, sample_model.json is pk 1, and so on. `loaddata` UPDATES the row
+# at each id rather than skipping it, so running these against a populated
+# database silently overwrites live records.
+#
+# That is a real hazard for this deployment: HSEO's models are being migrated in
+# from NREL's instance, and if their reference data has drifted from the fixtures
+# in this repo, an unconditional load on every container boot would quietly
+# revert it. So we look before we write, and we fail closed if we cannot tell.
+#
+# To reseed on purpose, run the loaddata commands from the Getting Started docs
+# by hand.
+#
+set -euo pipefail
+
+SEEDED=$(python3 manage.py shell -c "
+from api.models.calliope import Abstract_Tech, Parameter
+from template.models import Template_Type
+seeded = (
+    Abstract_Tech.objects.exists()
+    or Parameter.objects.exists()
+    or Template_Type.objects.exists()
+)
+print('SEEDCHECK:' + ('yes' if seeded else 'no'))
+" 2>/dev/null | grep -o 'SEEDCHECK:[a-z]*' | tail -n 1 | cut -d: -f2)
+
+if [ "$SEEDED" = "yes" ]; then
+  echo "Reference data already present -- skipping fixture load to protect existing rows."
+  exit 0
+fi
+
+if [ "$SEEDED" != "no" ]; then
+  echo "Could not determine whether reference data exists. Refusing to load fixtures." >&2
+  echo "Fix the database connection, or load fixtures by hand once you have checked." >&2
+  exit 1
+fi
+
+echo "Empty database detected -- loading reference data."
+
+# api fixtures first: the template fixtures below reference api.Parameter and
+# api.Abstract_Tech rows by id, and template/models.py runs a pre_save hook that
+# looks them up. Load them the other way round and the whole batch rolls back.
+python3 manage.py loaddata --app api \
+      admin_abstract_tech.json \
+      admin_parameter.json \
+      admin_abstract_tech_param.json \
+      admin_run_parameter.json
+
+python3 manage.py loaddata --app template \
+      admin_template_type.json \
+      admin_template_type_variables.json \
+      admin_template_type_techs.json \
+      admin_template_type_locs.json \
+      admin_template_type_loc_techs.json \
+      admin_template_type_loc_tech_params.json \
+      admin_template_type_tech_params.json \
+      admin_template_type_carriers.json
+
+echo "Reference data loaded."

calliope_app/requirements.txt

@@ -7,11 +7,13 @@ django-crispy-forms==1.14.0
 django-environ>=0.4.5
 django-modeltranslation==0.18.12
 flower>=2.0.1
+gunicorn==21.2.0
 nrel-pysam==3.0.2
 pint==0.21
 psycopg2-binary==2.9.3
 pyyaml==6.0
 requests>=2.21.0
+whitenoise==6.6.0
 
 pyutilib>=6.0.0
 # amqp==2.6.1