feat(sync-secrets-with-s3): automatically update secrets with the buc…

…ket in order to communicate with the napse app

backend/config/settings/__init__.py

@@ -0,0 +1 @@
+from .db_essentials import sync_master_key_s3

backend/config/settings/db_essentials.py

@@ -0,0 +1,30 @@
+import json
+from time import sleep
+
+import environ
+from django.db.models.signals import post_migrate
+from django.dispatch import receiver
+from django_napse.core.settings import napse_settings
+from utils import sync_file_with_s3
+
+
+@receiver(post_migrate)
+def sync_master_key_s3(sender, **kwargs):
+    env = environ.Env()
+    if "AWS_ACCESS_KEY_ID" not in env or "AWS_SECRET_ACCESS_KEY" not in env or "AWS_S3_BUCKET_URI" not in env:
+        print("AWS credentials not found. Skipping sync_master_key_s3")
+        return
+    master_key_created = False
+    counter = 0
+    while not master_key_created:
+        with open(napse_settings.NAPSE_SECRETS_FILE_PATH, "r") as f:
+            data = json.load(f)
+        if "master_key" in data:
+            master_key_created = True
+        counter += 1
+        sleep(1)
+        print(f"Waiting for master key to be created. {counter} seconds passed")
+        if counter > 10:
+            error_msg = "Master key was not created in 10 seconds"
+            raise ValueError(error_msg)
+    sync_file_with_s3(napse_settings.NAPSE_SECRETS_FILE_PATH, "napse-eb-bucket", "napse-secrets.json")

backend/config/settings/production.py

@@ -12,7 +12,7 @@
 
 DATABASES = {
     "default": {
-        "ENGINE": "django.db.backends.sqlite3",
+        "ENGINE": "utils.sqlite3_with_busy_timeout.SqliteWithBusyTimeout",
         "NAME": ROOT_DIR / "db" / "db.sqlite3",
     },
 }

backend/docker/compose/production/django/start

@@ -4,7 +4,10 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
-
+if ( ! test -f /app/secrets.json ); then
+    echo "Secrets file not found, creating..."
+    touch /app/secrets.json
+fi
 cat << EOF > /app/secrets.json 
 {
     "Exchange Accounts": {

backend/requirements/base.txt

@@ -24,4 +24,4 @@ django-napse==1.10.4 # https://github.com/napse-invest/django-napse
 djangorestframework-api-key==2.3.0
 
 # External tools
-coverage==7.4.0 # https://github.com/nedbat/coveragepy
+boto3==1.34.14 # https://boto3.amazonaws.com/v1/documentation/api/latest/guide/quickstart.html

backend/requirements/development.txt

@@ -19,6 +19,7 @@ black==23.12.1  # https://github.com/psf/black
 pylint-django==2.5.5  # https://github.com/PyCQA/pylint-django
 pylint-celery==0.3  # https://github.com/PyCQA/pylint-celery
 pre-commit==3.6.0  # https://github.com/pre-commit/pre-commit
+coverage==7.4.0 # https://github.com/nedbat/coveragepy
 
 # Django
 # ------------------------------------------------------------------------------

backend/utils/__init__.py

@@ -0,0 +1 @@
+from .sysnc_file_with_s3 import sync_file_with_s3

backend/utils/sqlite3_with_busy_timeout.py

@@ -0,0 +1,8 @@
+from django.db.backends.sqlite3.base import SqliteDatabaseWrapper
+
+
+class SqliteWithBusyTimeout(SqliteDatabaseWrapper):
+    def _cursor(self):
+        cursor = super()._cursor()
+        cursor.execute("PRAGMA busy_timeout = 5000")
+        return cursor

backend/utils/sysnc_file_with_s3.py

@@ -0,0 +1,17 @@
+import json
+
+import boto3
+import environ
+
+
+def sync_file_with_s3(path: str, bucket_name: str, key: str):
+    env = environ.Env()
+    if "AWS_ACCESS_KEY_ID" not in env or "AWS_SECRET_ACCESS_KEY" not in env or "AWS_S3_BUCKET_URI" not in env:
+        error_msg = "AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY or AWS_S3_BUCKET_URI is not set in .env file"
+        raise ValueError(error_msg)
+
+    s3 = boto3.client("s3", aws_access_key_id=env.get_value("AWS_ACCESS_KEY_ID"), aws_secret_access_key=env.get_value("AWS_SECRET_ACCESS_KEY"))
+    with open(path, "r") as f:
+        data = json.load(f)
+    s3.put_object(Bucket=bucket_name, Key=key, Body=json.dumps(data))
+    print(f"Successfully synced {path} with S3 bucket {bucket_name} with key {key}")

pyproject.toml

@@ -19,12 +19,12 @@ ignore = [
     "RUF012", # Mutable class attributes should be annotated with `typing.ClassVar`
     ]
 
-# Allow autofix for all enabled rules (when `--fix`) is provided.
+# Allow autofix for all enabled rules (when `--fix`) is provided
 fixable = ["A", "B", "C", "D", "E", "F", "DTZ", "RUF", "S", "COM", "C4", "DJ", "EM", "ISC", "ICN", "PIE", "RET", "SLF", "SIM", "TID", "PD", "NPY"]
 unfixable = []
 
 # Exclude a variety of commonly ignored directories.
-exclude = [ 
+exclude = [
     ".bzr",
     ".direnv",
     ".eggs",
@@ -49,6 +49,7 @@ exclude = [
     "migrations",
 ]
 
+pydocstyle.convention ="google"
 # Same as Black.
 line-length = 150
 
@@ -63,4 +64,8 @@ target-version = "py311"
 
 [tool.ruff.mccabe]
 # Unlike Flake8, default to a complexity level of 10.
-max-complexity = 10
+max-complexity = 10
+
+[tool.ruff.format]
+docstring-code-format = true
+docstring-code-line-length = 60

setup-osx.sh

@@ -1,4 +1,5 @@
 brew install [email protected]
+python3.11 -m pip install --upgrade pip
 pip3 install virtualenv
 pip3 install pip-tools
 python3 -m virtualenv .venv --python=python3.11

setup-unix.sh

@@ -3,6 +3,7 @@ sudo apt update
 sudo apt install git-flow
 sudo apt install python3
 sudo apt install python3.11-dev
+python3.11 -m pip install --upgrade pip
 pip3 install --upgrade pip
 pip install virtualenv