TP2000-1463 Support SQLite dump on DBT Platform

common/jinja2/common/app_info.jinja

@@ -52,9 +52,9 @@
           {"text": "Environment variable"},
         ],
         [
-          {"text": "APP_UPDATED_TIME"},
-          {"text": APP_UPDATED_TIME},
-          {"text": "Estimated application deploy time"},
+          {"text": "UPTIME"},
+          {"text": UPTIME},
+          {"text": "Time this instance has been in service"},
         ],
         [
           {"text": "LAST_TRANSACTION_TIME"},

common/tests/test_util.py

@@ -1,3 +1,4 @@
+import json
 import os
 from unittest import mock
 
@@ -14,6 +15,64 @@
 pytestmark = pytest.mark.django_db
 
 
+@pytest.mark.parametrize(
+    "environment_key, expected_result",
+    (
+        (
+            {
+                "engine": "engine",
+                "username": "username",
+                "password": "password",
+                "host": "host",
+                "port": 1234,
+                "dbname": "dbname",
+            },
+            "engine://username:password@host:1234/dbname",
+        ),
+        (
+            {
+                "engine": "engine",
+                "username": "username",
+                "host": "host",
+                "dbname": "dbname",
+            },
+            "engine://username@host/dbname",
+        ),
+        (
+            {
+                "engine": "engine",
+                "host": "host",
+                "dbname": "dbname",
+            },
+            "engine://host/dbname",
+        ),
+        (
+            {
+                "engine": "engine",
+                "password": "password",
+                "port": 1234,
+                "dbname": "dbname",
+            },
+            "engine:///dbname",
+        ),
+        (
+            {
+                "engine": "engine",
+                "dbname": "dbname",
+            },
+            "engine:///dbname",
+        ),
+    ),
+)
+def test_database_url_from_env(environment_key, expected_result):
+    with mock.patch.dict(
+        os.environ,
+        {"DATABASE_CREDENTIALS": json.dumps(environment_key)},
+        clear=True,
+    ):
+        assert util.database_url_from_env("DATABASE_CREDENTIALS") == expected_result
+
+
 @pytest.mark.parametrize(
     "value, expected",
     [

common/util.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import json
 import os
 import re
 import typing
@@ -54,10 +55,53 @@
 major, minor, patch = python_version_tuple()
 
 
+def is_cloud_foundry():
+    """Return True if the deployment environment contains a `VCAP_SERVICES` env
+    var, indicating a CloudFoundry environment, False otherwise."""
+    return "VCAP_SERVICES" in os.environ
+
+
 def classproperty(fn):
     return classmethod(property(fn))
 
 
+def database_url_from_env(environment_key: str) -> str:
+    """
+    Return a database URL string from the environment variable identified by
+    `environment_key`. The environment variable should be parsable as a
+    JSON-like string and may contain the keys:
+
+        "engine" (Required) - database engine id. For instance "postgres" or "sqlite".
+        "username" (Optional if "password" is not present) - database user name.
+        "password" (Optional) - database user's password.
+        "host" (Optional if "port" is not present) - database hostname.
+        "port" (Optional) - database host port.
+        "dbname" (Required) - database name.
+
+    If all keys are present, then the returned result would be a string of the
+    form:
+
+        <engine>://<username>:<password>@<host>:<port>/<dbname>
+
+    This is a plug-in, less naive version of
+    `dbt_copilot_python.database.database_url_from_env()` making `username`,
+    `password`, `host` and `port` an optional as described above.
+    """
+    config = json.loads(os.environ[environment_key])
+
+    username = config.get("username", "")
+    password = config.get("password")
+    host = config.get("host", "")
+    port = config.get("port")
+
+    config["username"] = username
+    config["password"] = f":{password}" if username and password else ""
+    config["host"] = f"@{host}" if (username or password) and host else host
+    config["port"] = f":{port}" if host and port else ""
+
+    return "{engine}://{username}{password}{host}{port}/{dbname}".format(**config)
+
+
 def is_truthy(value: Union[str, bool]) -> bool:
     """
     Check whether a string represents a True boolean value.

common/views/pages.py

@@ -1,8 +1,10 @@
 """Common views."""
 
+import logging
 import os
 import time
 from datetime import datetime
+from datetime import timedelta
 from typing import Dict
 from typing import List
 from typing import Optional
@@ -36,6 +38,7 @@
 from common.celery import app as celery_app
 from common.forms import HomeSearchForm
 from common.models import Transaction
+from common.util import is_cloud_foundry
 from exporter.sqlite.util import sqlite_dumps
 from footnotes.models import Footnote
 from geo_areas.models import GeographicalArea
@@ -47,6 +50,8 @@
 from workbaskets.models import WorkBasket
 from workbaskets.models import WorkflowStatus
 
+logger = logging.getLogger(__name__)
+
 
 class HomeView(LoginRequiredMixin, FormView):
     template_name = "common/homepage.jinja"
@@ -322,6 +327,33 @@ def get(self, request, *args, **kwargs) -> HttpResponse:
         )
 
 
+def get_uptime() -> str:
+    """
+    Return approximate system uptime in a platform-independent way as a string
+    in the following format:
+        "<days> days, <hours> hours, <mins> minutes"
+    """
+    try:
+        if is_cloud_foundry():
+            # CF recycles Garden containers so time.monotonic() returns a
+            # misleading value. However, file modified time is set on deployment.
+            uptime = timedelta(seconds=(time.time() - os.path.getmtime(__file__)))
+        else:
+            # time.monotonic() doesn't count time spent in hibernation, so may
+            # be inaccurate on systems that hibernate.
+            uptime = timedelta(seconds=time.monotonic())
+
+        formatted_uptime = (
+            f"{uptime.days} days, {uptime.seconds // 3600} hours, "
+            f"{uptime.seconds // 60 % 60} minutes"
+        )
+    except Exception as e:
+        logger.error(e)
+        formatted_uptime = "Error getting uptime"
+
+    return formatted_uptime
+
+
 class AppInfoView(
     LoginRequiredMixin,
     TemplateView,
@@ -416,9 +448,7 @@ def get_context_data(self, **kwargs):
         if self.request.user.is_superuser:
             data["GIT_BRANCH"] = os.getenv("GIT_BRANCH", "Unavailable")
             data["GIT_COMMIT"] = os.getenv("GIT_COMMIT", "Unavailable")
-            data["APP_UPDATED_TIME"] = AppInfoView.timestamp_to_datetime_string(
-                os.path.getmtime(__file__),
-            )
+            data["UPTIME"] = get_uptime()
             last_transaction = Transaction.objects.order_by("updated_at").last()
             data["LAST_TRANSACTION_TIME"] = (
                 format(

exporter/sqlite/__init__.py

@@ -73,8 +73,8 @@ def make_export(connection: apsw.Connection):
             Path(temp_sqlite_db.name),
         )
         plan = make_export_plan(plan_runner)
-        # make_tamato_database() creates a Connection instance that needs
-        # closing once an in-memory plan has been created from it.
+        # Runner.make_tamato_database() (above) creates a Connection instance
+        # that needs closing once an in-memory plan has been created from it.
         plan_runner.database.close()
 
     export_runner = runner.Runner(connection)