diff --git a/common/jinja2/common/app_info.jinja b/common/jinja2/common/app_info.jinja index 5dfd98afd..c462ec71b 100644 --- a/common/jinja2/common/app_info.jinja +++ b/common/jinja2/common/app_info.jinja @@ -52,9 +52,9 @@ {"text": "Environment variable"}, ], [ - {"text": "APP_UPDATED_TIME"}, - {"text": APP_UPDATED_TIME}, - {"text": "Estimated application deploy time"}, + {"text": "UPTIME"}, + {"text": UPTIME}, + {"text": "Time this instance has been in service"}, ], [ {"text": "LAST_TRANSACTION_TIME"}, diff --git a/common/tests/test_util.py b/common/tests/test_util.py index 66ed96f2a..ba956d454 100644 --- a/common/tests/test_util.py +++ b/common/tests/test_util.py @@ -1,3 +1,4 @@ +import json import os from unittest import mock @@ -14,6 +15,64 @@ pytestmark = pytest.mark.django_db +@pytest.mark.parametrize( + "environment_key, expected_result", + ( + ( + { + "engine": "engine", + "username": "username", + "password": "password", + "host": "host", + "port": 1234, + "dbname": "dbname", + }, + "engine://username:password@host:1234/dbname", + ), + ( + { + "engine": "engine", + "username": "username", + "host": "host", + "dbname": "dbname", + }, + "engine://username@host/dbname", + ), + ( + { + "engine": "engine", + "host": "host", + "dbname": "dbname", + }, + "engine://host/dbname", + ), + ( + { + "engine": "engine", + "password": "password", + "port": 1234, + "dbname": "dbname", + }, + "engine:///dbname", + ), + ( + { + "engine": "engine", + "dbname": "dbname", + }, + "engine:///dbname", + ), + ), +) +def test_database_url_from_env(environment_key, expected_result): + with mock.patch.dict( + os.environ, + {"DATABASE_CREDENTIALS": json.dumps(environment_key)}, + clear=True, + ): + assert util.database_url_from_env("DATABASE_CREDENTIALS") == expected_result + + @pytest.mark.parametrize( "value, expected", [ diff --git a/common/util.py b/common/util.py index 7f06fb9c1..667f96b79 100644 --- a/common/util.py +++ b/common/util.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import os import re import typing @@ -54,10 +55,53 @@ major, minor, patch = python_version_tuple() +def is_cloud_foundry(): + """Return True if the deployment environment contains a `VCAP_SERVICES` env + var, indicating a CloudFoundry environment, False otherwise.""" + return "VCAP_SERVICES" in os.environ + + def classproperty(fn): return classmethod(property(fn)) +def database_url_from_env(environment_key: str) -> str: + """ + Return a database URL string from the environment variable identified by + `environment_key`. The environment variable should be parsable as a + JSON-like string and may contain the keys: + + "engine" (Required) - database engine id. For instance "postgres" or "sqlite". + "username" (Optional if "password" is not present) - database user name. + "password" (Optional) - database user's password. + "host" (Optional if "port" is not present) - database hostname. + "port" (Optional) - database host port. + "dbname" (Required) - database name. + + If all keys are present, then the returned result would be a string of the + form: + + ://:@:/ + + This is a plug-in, less naive version of + `dbt_copilot_python.database.database_url_from_env()` making `username`, + `password`, `host` and `port` an optional as described above. + """ + config = json.loads(os.environ[environment_key]) + + username = config.get("username", "") + password = config.get("password") + host = config.get("host", "") + port = config.get("port") + + config["username"] = username + config["password"] = f":{password}" if username and password else "" + config["host"] = f"@{host}" if (username or password) and host else host + config["port"] = f":{port}" if host and port else "" + + return "{engine}://{username}{password}{host}{port}/{dbname}".format(**config) + + def is_truthy(value: Union[str, bool]) -> bool: """ Check whether a string represents a True boolean value. diff --git a/common/views/pages.py b/common/views/pages.py index 875d45e85..23259ddf8 100644 --- a/common/views/pages.py +++ b/common/views/pages.py @@ -1,8 +1,10 @@ """Common views.""" +import logging import os import time from datetime import datetime +from datetime import timedelta from typing import Dict from typing import List from typing import Optional @@ -36,6 +38,7 @@ from common.celery import app as celery_app from common.forms import HomeSearchForm from common.models import Transaction +from common.util import is_cloud_foundry from exporter.sqlite.util import sqlite_dumps from footnotes.models import Footnote from geo_areas.models import GeographicalArea @@ -47,6 +50,8 @@ from workbaskets.models import WorkBasket from workbaskets.models import WorkflowStatus +logger = logging.getLogger(__name__) + class HomeView(LoginRequiredMixin, FormView): template_name = "common/homepage.jinja" @@ -322,6 +327,33 @@ def get(self, request, *args, **kwargs) -> HttpResponse: ) +def get_uptime() -> str: + """ + Return approximate system uptime in a platform-independent way as a string + in the following format: + " days, hours, minutes" + """ + try: + if is_cloud_foundry(): + # CF recycles Garden containers so time.monotonic() returns a + # misleading value. However, file modified time is set on deployment. + uptime = timedelta(seconds=(time.time() - os.path.getmtime(__file__))) + else: + # time.monotonic() doesn't count time spent in hibernation, so may + # be inaccurate on systems that hibernate. + uptime = timedelta(seconds=time.monotonic()) + + formatted_uptime = ( + f"{uptime.days} days, {uptime.seconds // 3600} hours, " + f"{uptime.seconds // 60 % 60} minutes" + ) + except Exception as e: + logger.error(e) + formatted_uptime = "Error getting uptime" + + return formatted_uptime + + class AppInfoView( LoginRequiredMixin, TemplateView, @@ -416,9 +448,7 @@ def get_context_data(self, **kwargs): if self.request.user.is_superuser: data["GIT_BRANCH"] = os.getenv("GIT_BRANCH", "Unavailable") data["GIT_COMMIT"] = os.getenv("GIT_COMMIT", "Unavailable") - data["APP_UPDATED_TIME"] = AppInfoView.timestamp_to_datetime_string( - os.path.getmtime(__file__), - ) + data["UPTIME"] = get_uptime() last_transaction = Transaction.objects.order_by("updated_at").last() data["LAST_TRANSACTION_TIME"] = ( format( diff --git a/exporter/sqlite/__init__.py b/exporter/sqlite/__init__.py index e85ac64c1..0450c3a21 100644 --- a/exporter/sqlite/__init__.py +++ b/exporter/sqlite/__init__.py @@ -73,8 +73,8 @@ def make_export(connection: apsw.Connection): Path(temp_sqlite_db.name), ) plan = make_export_plan(plan_runner) - # make_tamato_database() creates a Connection instance that needs - # closing once an in-memory plan has been created from it. + # Runner.make_tamato_database() (above) creates a Connection instance + # that needs closing once an in-memory plan has been created from it. plan_runner.database.close() export_runner = runner.Runner(connection) diff --git a/exporter/sqlite/runner.py b/exporter/sqlite/runner.py index 070a87cf2..ae31bb75b 100644 --- a/exporter/sqlite/runner.py +++ b/exporter/sqlite/runner.py @@ -1,9 +1,11 @@ import json import logging import os +import shutil +import subprocess import sys from pathlib import Path -from subprocess import run +from tempfile import TemporaryDirectory from typing import Iterable from typing import Iterator from typing import Tuple @@ -16,80 +18,203 @@ logger = logging.getLogger(__name__) -class Runner: - """Runs commands on an SQLite database.""" +def normalise_loglevel(loglevel): + """ + Attempt conversion of `loglevel` from a string integer value (e.g. "20") to + its loglevel name (e.g. "INFO"). - database: apsw.Connection + This function can be used after, for instance, copying log levels from + environment variables, when the incorrect representation (int as string + rather than the log level name) may occur. + """ + try: + return logging._levelToName.get(int(loglevel)) + except: + return loglevel - def __init__(self, database: apsw.Connection) -> None: - self.database = database - @classmethod - def normalise_loglevel(cls, loglevel): - """ - Attempt conversion of `loglevel` from a string integer value (e.g. "20") - to its loglevel name (e.g. "INFO"). +SQLITE_MIGRATIONS_NAME = "sqlite_export" +"""Name passed to `manage.py makemigrations`, via the --name flag, when creating +the SQLite migrations source files.""" - This function can be used after, for instance, copying log levels from - environment variables, when the incorrect representation (int as string - rather than the log level name) may occur. - """ - try: - return logging._levelToName.get(int(loglevel)) - except: - return loglevel +SQLITE_MIGRATIONS_GLOB = f"**/migrations/*{SQLITE_MIGRATIONS_NAME}.py" +"""Glob pattern matching all SQLite-specific migration source files generated by +the `manage.py makemigrations --name sqlite_export` command.""" - @classmethod - def manage(cls, sqlite_file: Path, *args: str): + +class SQLiteMigrationCurrentDirectory: + """ + Context manager class that uses the application's current base directory for + managing SQLite migrations. + + Upon exiting the context manager, SQLite-specific migration files are + deleted. + """ + + def __enter__(self): + logger.info(f"Entering context manager {self.__class__.__name__}") + return settings.BASE_DIR + + def __exit__(self, exc_type, exc_value, traceback): + logger.info(f"Exiting context manager {self.__class__.__name__}") + for file in Path(settings.BASE_DIR).rglob(SQLITE_MIGRATIONS_GLOB): + file.unlink() + + +class SQLiteMigrationTemporaryDirectory(TemporaryDirectory): + """ + Context manager class that provides a newly created temporary directory + (under the OS's temporary directory system) for managing SQLite migrations. + + Upon exiting the context manager, the temporary directory is deleted. + """ + + def __enter__(self): + logger.info(f"Entering context manager {self.__class__.__name__}") + + tmp_dir = super().__enter__() + tmp_dir = os.path.join(tmp_dir, "tamato_sqlite_migration") + shutil.copytree(settings.BASE_DIR, tmp_dir) + + # Ensure migrations directories are writable to allow SQLite migrations + # to be created - some deployments make source tree directories + # non-wriable. + for d in [p for p in Path(tmp_dir).rglob("migrations") if p.is_dir()]: + d.chmod(0o777) + + copied_files = [f for f in Path(tmp_dir).rglob("*") if f.is_file()] + logger.info(f"Copied {len(copied_files)} files to {tmp_dir}") + + return tmp_dir + + def __exit__(self, exc_type, exc_value, traceback): + logger.info(f"Exiting context manager {self.__class__.__name__}") + super().__exit__(exc_type, exc_value, traceback) + + +class SQLiteMigrator: + """ + Populates a new and empty SQLite database file with the Tamato database + schema derived from Tamato's models. + + This is required because SQLite uses different fields to PostgreSQL, missing + migrations are first generated to bring in the different style of validity + fields. + + This is done by creating additional, auxiliary migrations that are specific + to the SQLite and then executing them to populate the database with the + schema. + """ + + sqlite_file: Path + + def __init__(self, sqlite_file: Path, migrations_in_tmp_dir=False): + self.sqlite_file = sqlite_file + self.migration_directory_class = ( + SQLiteMigrationTemporaryDirectory + if migrations_in_tmp_dir + else SQLiteMigrationCurrentDirectory + ) + + def migrate(self): + from manage import ENV_INFO_FLAG + + with self.migration_directory_class() as migration_dir: + logger.info(f"Running `makemigrations` in {migration_dir}") + self.manage( + migration_dir, + ENV_INFO_FLAG, + "makemigrations", + "--name", + SQLITE_MIGRATIONS_NAME, + ) + + sqlite_migration_files = [ + f + for f in Path(migration_dir).rglob(SQLITE_MIGRATIONS_GLOB) + if f.is_file() + ] + logger.info( + f"{len(sqlite_migration_files)} SQLite migration files " + f"generated in {migration_dir}", + ) + + logger.info(f"Running `migrate` in {migration_dir}") + self.manage( + migration_dir, + ENV_INFO_FLAG, + "migrate", + ) + + def manage(self, exec_dir: str, *manage_args: str): """ Runs a Django management command on the SQLite database. This management command will be run such that ``settings.SQLITE`` is True, allowing SQLite specific functionality to be switched on and off using the value of this setting. + + `exec_dir` sets the directory in which the management command should be + executed. """ + sqlite_env = os.environ.copy() # Correct log levels that are incorrectly expressed as string ints. if "CELERY_LOG_LEVEL" in sqlite_env: - sqlite_env["CELERY_LOG_LEVEL"] = cls.normalise_loglevel( + sqlite_env["CELERY_LOG_LEVEL"] = normalise_loglevel( sqlite_env["CELERY_LOG_LEVEL"], ) - sqlite_env["DATABASE_URL"] = f"sqlite:///{str(sqlite_file)}" - # Required to make sure the postgres default isn't set as the DB_URL + # Set up environment-specific env var values. if sqlite_env.get("VCAP_SERVICES"): vcap_env = json.loads(sqlite_env["VCAP_SERVICES"]) vcap_env.pop("postgres", None) sqlite_env["VCAP_SERVICES"] = json.dumps(vcap_env) + sqlite_env["DATABASE_URL"] = f"sqlite:///{str(self.sqlite_file)}" + elif sqlite_env.get("COPILOT_ENVIRONMENT_NAME"): + sqlite_env["DATABASE_CREDENTIALS"] = json.dumps( + { + "engine": "sqlite", + "dbname": f"{str(self.sqlite_file)}", + }, + ) + else: + sqlite_env["DATABASE_URL"] = f"sqlite:///{str(self.sqlite_file)}" - run( - [sys.executable, "manage.py", *args], - cwd=settings.BASE_DIR, - capture_output=False, + sqlite_env["PATH"] = exec_dir + ":" + sqlite_env["PATH"] + manage_cmd = os.path.join(exec_dir, "manage.py") + + subprocess.run( + [sys.executable, manage_cmd, *manage_args], + cwd=exec_dir, + check=True, env=sqlite_env, ) + +class Runner: + """Runs commands on an SQLite database.""" + + database: apsw.Connection + + def __init__(self, database: apsw.Connection) -> None: + self.database = database + @classmethod def make_tamato_database(cls, sqlite_file: Path) -> "Runner": """Generate a new and empty SQLite database with the TaMaTo schema derived from Tamato's models - by performing 'makemigrations' followed by 'migrate' on the Sqlite file located at `sqlite_file`.""" - try: - # Because SQLite uses different fields to PostgreSQL, missing - # migrations are first generated to bring in the different style of - # validity fields. However, these should not be applied to Postgres - # and so should be removed (in the `finally` block) after they have - # been applied (when running `migrate`). - cls.manage(sqlite_file, "makemigrations", "--name", "sqlite_export") - cls.manage(sqlite_file, "migrate") - assert sqlite_file.exists() - return cls(apsw.Connection(str(sqlite_file))) - finally: - for file in Path(settings.BASE_DIR).rglob( - "**/migrations/*sqlite_export.py", - ): - file.unlink() + + sqlite_migrator = SQLiteMigrator( + sqlite_file=sqlite_file, + migrations_in_tmp_dir=settings.SQLITE_MIGRATIONS_IN_TMP_DIR, + ) + sqlite_migrator.migrate() + + assert sqlite_file.exists() + return cls(apsw.Connection(str(sqlite_file))) def read_schema(self, type: str) -> Iterator[Tuple[str, str]]: """ diff --git a/exporter/storages.py b/exporter/storages.py index a6a86ac7d..6b2c65d0a 100644 --- a/exporter/storages.py +++ b/exporter/storages.py @@ -1,4 +1,5 @@ import logging +import sqlite3 from functools import cached_property from os import path from pathlib import Path @@ -15,6 +16,40 @@ logger = logging.getLogger(__name__) +class EmptyFileException(Exception): + pass + + +def is_valid_sqlite(file_path: str) -> bool: + """ + `file_path` should be a path to a file on the local file system. Validation. + + includes: + - test that a file exists at `file_path`, + - test that the file at `file_path` has non-zero size, + - perform a SQLite PRAGMA quick_check on file at `file_path`. + + If errors are found during validation, then exceptions that this function + may raise include: + - sqlite3.DatabaseError if the PRAGMA quick_check fails. + - FileNotFoundError if no file was found at `file_path`. + - exporter.storage.EmptyFileException if the file at `file_path` has + zero size. + + Returns True if validation checks all pass. + """ + + if path.getsize(file_path) == 0: + raise EmptyFileException(f"{file_path} has zero size.") + + with sqlite3.connect(file_path) as connection: + cursor = connection.cursor() + # Executing "PRAGMA quick_check" raises DatabaseError if the SQLite + # database file is invalid. + cursor.execute("PRAGMA quick_check") + return True + + class HMRCStorage(S3Boto3Storage): def get_default_settings(self): # Importing settings here makes it possible for tests to override_settings @@ -113,7 +148,9 @@ def export_database(self, filename: str): sqlite.make_export(connection) connection.close() logger.info(f"Saving {filename} to S3 storage.") - self.save(filename, temp_sqlite_db.file) + if is_valid_sqlite(temp_sqlite_db.name): + # Only save to S3 if the SQLite file is valid. + self.save(filename, temp_sqlite_db.file) class SQLiteLocalStorage(SQLiteExportMixin, Storage): diff --git a/exporter/tests/test_files/empty_sqlite.db b/exporter/tests/test_files/empty_sqlite.db new file mode 100644 index 000000000..e69de29bb diff --git a/exporter/tests/test_files/invalid_sqlite.db b/exporter/tests/test_files/invalid_sqlite.db new file mode 100644 index 000000000..abe95ec8e --- /dev/null +++ b/exporter/tests/test_files/invalid_sqlite.db @@ -0,0 +1 @@ +invalid sqlite file content \ No newline at end of file diff --git a/exporter/tests/test_files/valid_sqlite.db b/exporter/tests/test_files/valid_sqlite.db new file mode 100644 index 000000000..8c92662d4 Binary files /dev/null and b/exporter/tests/test_files/valid_sqlite.db differ diff --git a/exporter/tests/test_sqlite.py b/exporter/tests/test_sqlite.py index c75d8f839..bb4f7cdf9 100644 --- a/exporter/tests/test_sqlite.py +++ b/exporter/tests/test_sqlite.py @@ -1,4 +1,6 @@ +import sqlite3 import tempfile +from contextlib import nullcontext from io import BytesIO from os import path from pathlib import Path @@ -13,6 +15,9 @@ from exporter.sqlite import plan from exporter.sqlite import tasks from exporter.sqlite.runner import Runner +from exporter.sqlite.runner import SQLiteMigrator +from exporter.storages import EmptyFileException +from exporter.storages import is_valid_sqlite from workbaskets.validators import WorkflowStatus pytestmark = pytest.mark.django_db @@ -42,6 +47,58 @@ def sqlite_database(sqlite_template: Runner) -> Iterator[Runner]: yield Runner(in_memory_database) +def get_test_file_path(filename): + return path.join( + path.dirname(path.abspath(__file__)), + "test_files", + filename, + ) + + +@pytest.mark.parametrize( + ("test_file_path, expect_context"), + ( + ( + get_test_file_path("valid_sqlite.db"), + nullcontext(), + ), + ( + "/invalid/file/path", + pytest.raises(FileNotFoundError), + ), + ( + get_test_file_path("empty_sqlite.db"), + pytest.raises(EmptyFileException), + ), + ( + get_test_file_path("invalid_sqlite.db"), + pytest.raises(sqlite3.DatabaseError), + ), + ), +) +def test_is_valid_sqlite(test_file_path, expect_context): + """Test that `is_valid_sqlite()` raises correct exceptions for invalid + SQLite files and succeeds for valid SQLite files.""" + with expect_context: + is_valid_sqlite(test_file_path) + + +@pytest.mark.parametrize( + ("migrations_in_tmp_dir"), + (False, True), +) +def test_sqlite_migrator(migrations_in_tmp_dir): + """Test SQLiteMigrator.""" + with tempfile.NamedTemporaryFile() as sqlite_file: + sqlite_migrator = SQLiteMigrator( + sqlite_file=Path(sqlite_file.name), + migrations_in_tmp_dir=migrations_in_tmp_dir, + ) + sqlite_migrator.migrate() + + assert is_valid_sqlite(sqlite_file.name) + + FACTORIES_EXPORTED = [ factory for factory in factories.TrackedModelMixin.__subclasses__() diff --git a/hmrc_sdes/tests/test_client.py b/hmrc_sdes/tests/test_client.py index ebd660b11..9533729dc 100644 --- a/hmrc_sdes/tests/test_client.py +++ b/hmrc_sdes/tests/test_client.py @@ -63,7 +63,9 @@ def test_api_call(responses, settings): responses.add_passthru(settings.HMRC["base_url"]) # reload settings from env, overriding test settings - dotenv.read_dotenv(os.path.join(settings.BASE_DIR, ".env")) + import dotenv + + dotenv.load_dotenv(dot_envpath=os.path.join(settings.BASE_DIR, ".env")) settings.HMRC["client_id"] = os.environ.get("HMRC_API_CLIENT_ID") settings.HMRC["client_secret"] = os.environ.get("HMRC_API_CLIENT_SECRET") settings.HMRC["service_reference_number"] = os.environ.get( diff --git a/manage.py b/manage.py index 6122f2dd9..d101448f0 100755 --- a/manage.py +++ b/manage.py @@ -6,14 +6,52 @@ import dotenv +ENV_INFO_FLAG = "--env-info" + + +def output_env_info(): + """Inspect and output environment diagnostics for help with platform / + environment debugging.""" + + import pwd + from pathlib import Path + + cwd = Path().resolve() + script_path = Path(__file__).resolve() + executable_path = Path(sys.executable).resolve() + path = os.environ.get("PATH") + username = pwd.getpwuid(os.getuid()).pw_name + + print("Environment diagnostics") + print("----") + print(f" Current working directory: {cwd}") + print(f" Current script path: {script_path}") + print(f" Python executable path: {executable_path}") + print(f" PATH: {path}") + print(f" username: {username}") + print("----") + + # Remove the flag to avoid Django unknown command errors. + sys.argv = [arg for arg in sys.argv if arg != ENV_INFO_FLAG] + + +def set_django_settings_module(): + """Set the DJANGO_SETTINGS_MODULE env var with an appropriate value.""" -def main(): in_test = not {"pytest", "test"}.isdisjoint(sys.argv[1:]) in_dev = in_test is False and "DEV" == str(os.environ.get("ENV")).upper() os.environ.setdefault( "DJANGO_SETTINGS_MODULE", "settings.test" if in_test else "settings.dev" if in_dev else "settings", ) + + +def main(): + if ENV_INFO_FLAG in sys.argv: + output_env_info() + + set_django_settings_module() + try: from django.core.management import execute_from_command_line except ImportError as exc: @@ -28,5 +66,5 @@ def main(): if __name__ == "__main__": with warnings.catch_warnings(): warnings.simplefilter("ignore") - dotenv.read_dotenv() + dotenv.load_dotenv() main() diff --git a/pyproject.toml b/pyproject.toml index 127fb30e6..fbd3393b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,6 @@ readme = "README.md" dependencies = [ "dj-database-url", "django", - "django-dotenv", "django-extra-fields", "django-filter", "django-fsm", @@ -28,6 +27,7 @@ dependencies = [ "gunicorn", "jinja2", "psycopg[binary]", + "python-dotenv", "sentry-sdk", "werkzeug", "whitenoise", diff --git a/requirements-dev-jupyter.txt b/requirements-dev-jupyter.txt index ab997a0de..b9d584e35 100644 --- a/requirements-dev-jupyter.txt +++ b/requirements-dev-jupyter.txt @@ -1,6 +1,5 @@ -r requirements-dev.txt -ipython==8.18.1 +dj-notebook==0.7.0 +ipython==8.26.0 jupyter==1.0.0 -jupyter-nbextensions-configurator==0.6.3 -notebook==6.5.6 diff --git a/requirements.txt b/requirements.txt index ddcefd61b..3d434b6d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,6 @@ django-chunk-upload-handlers==0.0.13 django-crispy-forms==1.14.0 django-csp==3.6 django-cte==1.3.1 -django-dotenv==1.4.2 django-extensions==3.2.3 django-filter==23.5 django-formtools==2.3 @@ -67,6 +66,7 @@ pytest-forked==1.4.0 pytest-responses==0.5.0 pytest-xdist==2.5.0 pytest==7.4.0 +python-dotenv==1.0.1 python-magic==0.4.25 requests-oauthlib==1.3.0 requests-mock==1.10.0 diff --git a/settings/common.py b/settings/common.py index bbf3cdb60..da592d65a 100644 --- a/settings/common.py +++ b/settings/common.py @@ -12,12 +12,12 @@ import dj_database_url from celery.schedules import crontab -from dbt_copilot_python.database import database_url_from_env from dbt_copilot_python.network import setup_allowed_hosts from dbt_copilot_python.utility import is_copilot from django.urls import reverse_lazy from django_log_formatter_asim import ASIMFormatter +from common.util import database_url_from_env from common.util import is_truthy # Name of the deployment environment (dev/alpha) @@ -319,9 +319,9 @@ # -- Database + if MAINTENANCE_MODE: DATABASES = {} - # DBT PaaS elif is_copilot(): DB_URL = database_url_from_env("DATABASE_CREDENTIALS") @@ -527,6 +527,7 @@ if is_copilot(): SQLITE_S3_ACCESS_KEY_ID = None SQLITE_S3_SECRET_ACCESS_KEY = None + SQLITE_MIGRATIONS_IN_TMP_DIR = True else: SQLITE_S3_ACCESS_KEY_ID = os.environ.get( "SQLITE_S3_ACCESS_KEY_ID", @@ -536,6 +537,9 @@ "SQLITE_S3_SECRET_ACCESS_KEY", "test_sqlite_key", ) + SQLITE_MIGRATIONS_IN_TMP_DIR = is_truthy( + os.environ.get("SQLITE_MIGRATIONS_IN_TMP_DIR", False), + ) SQLITE_STORAGE_BUCKET_NAME = os.environ.get("SQLITE_STORAGE_BUCKET_NAME", "sqlite") SQLITE_S3_ENDPOINT_URL = os.environ.get( diff --git a/setup.py b/setup.py index 4dd71a5aa..93b4fde7b 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,6 @@ install_requires=[ "dj-database-url", "django", - "django-dotenv", "django-extra-fields", "django-filter", "django-fsm", @@ -26,6 +25,7 @@ "gunicorn", "jinja2", "psycopg[binary]", + "python-dotenv", "sentry-sdk", "werkzeug", "whitenoise", diff --git a/wsgi.py b/wsgi.py index 92efa35aa..e537f5824 100644 --- a/wsgi.py +++ b/wsgi.py @@ -13,7 +13,7 @@ from django.core.wsgi import get_wsgi_application -dotenv.read_dotenv() +dotenv.load_dotenv() os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")