feat: token auth with per-country RBAC (#6)

* feat: setup database & initial models

* feat: placeholder api key routes

* feat: use async pg driver; add hashing packages

* feat: initial routes to create/view/revoke api keys

* feat: working auth

* feat: use api key-secret combination instead

* feat: script to generate roles

* refactor: refine db query scalar

* refactor: optimize api key routes; bump to v0.2.0

* fix: auth scheme dependency injection

* docs: update .env.example

* build: add postgres & init containers to helm chart

* feat: filter countries based on role

* ci: update secrets

* refactor: address comments

.env.example

@@ -1,9 +1,20 @@
 DELTA_BEARER_TOKEN="<openssl rand -base64 32>"
+DELTA_SHARING_HOST="delta:8890"
+SECRET_KEY="<openssl rand -base64 64>"
+APP_DOMAIN="localhost:5000"
+
 SAS_TOKEN="get from Azure portal"
 STORAGE_ACCESS_KEY="get from Azure portal"
-STORAGE_ACCOUNT_NAME=
-CONTAINER_NAME=
-DELTA_SHARING_HOST="sharing-server:8890"
+STORAGE_ACCOUNT_NAME="get from Azure portal"
+CONTAINER_NAME="get from Azure portal"
 
 DATA_SHARING_PROXY_IMAGE_REPOSITORY=
 DATA_SHARING_DELTA_IMAGE_REPOSITORY=
+
+POSTGRESQL_USERNAME=
+POSTGRESQL_PASSWORD=
+POSTGRESQL_DATABASE=
+DB_HOST=db
+
+ADMIN_API_KEY="<poetry run python -c 'from uuid import uuid4; print(uuid4())'>"
+ADMIN_API_SECRET="<poetry run python -c 'from secrets import token_urlsafe; print(token_urlsafe(48))'>"

.flake8

@@ -2,3 +2,5 @@
 ignore = E203, E266, E501, W503
 max-line-length = 88
 max-complexity = 18
+per-file-ignores =
+    **/__init__.py:F401

Taskfile.yml

@@ -108,6 +108,7 @@ tasks:
     cmds:
       - kubectl apply -f k8s/configmap.yaml
       - kubectl apply -f k8s/secrets.yaml
+      - helm dependency update helm/data-sharing-proxy
       - >
         helm upgrade --install data-sharing-delta ./helm/data-sharing-delta
         --set image.repository={{.DATA_SHARING_DELTA_IMAGE_REPOSITORY}}
@@ -116,6 +117,7 @@ tasks:
         helm upgrade --install data-sharing-proxy ./helm/data-sharing-proxy
         --set image.repository={{.DATA_SHARING_PROXY_IMAGE_REPOSITORY}}
         --set image.tag={{.SHORT_SHA}}
+        --set service.type=LoadBalancer
 
   helm-down:
     desc: Uninstall Helm chart
@@ -129,4 +131,24 @@ tasks:
   generate-delta-config:
     desc: Generate Delta Sharing config
     cmds:
-      - poetry run python -m scripts.generate_delta_config
+      - task exec -- proxy python -m scripts.generate_delta_config
+
+  makemigrations:
+    desc: Generate database migrations
+    cmds:
+      - task exec -- proxy alembic revision --autogenerate -m "{{.CLI_ARGS}}"
+
+  migrate:
+    desc: Run database migrations
+    cmds:
+      - task exec -- proxy alembic upgrade head
+
+  psql:
+    desc: Connect to database
+    cmds:
+      - task exec -- db psql -U {{.POSTGRESQL_USERNAME}} -d {{.POSTGRESQL_DATABASE}}
+
+  load-fixtures:
+    desc: Install database fixtures
+    cmds:
+      - task exec -- proxy python -m scripts.load_fixtures {{.CLI_ARGS}}

alembic.ini

@@ -0,0 +1,116 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = data_sharing/migrations
+
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+# Uncomment the line below if you want the files to be prepended with date and time
+# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
+# for all available tokens
+# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python-dateutil library that can be
+# installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to data_sharing/migrations/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:data_sharing/migrations/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+
+# set to 'true' to search source files recursively
+# in each "version_locations" directory
+# new in Alembic version 1.10
+# recursive_version_locations = false
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+sqlalchemy.url = driver://user:pass@localhost/dbname
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# lint with attempts to fix using "ruff" - use the exec runner, execute a binary
+# hooks = ruff
+# ruff.type = exec
+# ruff.executable = %(here)s/.venv/bin/ruff
+# ruff.options = --fix REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

azure/azure-pipelines-dev.yaml

@@ -18,6 +18,11 @@ variables:
   storageAccessKey: $(STORAGE_ACCESS_KEY)
   storageAccountName: $(STORAGE_ACCOUNT_NAME)
   storageContainerName: $(STORAGE_CONTAINER_NAME)
+  postgresqlPassword: $(POSTGRESQL_PASSWORD)
+  adminApiKey: $(ADMIN_API_KEY)
+  adminApiSecret: $(ADMIN_API_SECRET)
+  secretKey: $(APPLICATION_SECRET_KEY)
+  appDomain: $(APP_DOMAIN)
   system.debug: true
 
 stages:
@@ -32,7 +37,7 @@ stages:
             deploy:
               steps:
                 - checkout: self
-                  
+
                 - task: Docker@2
                   displayName: Login to container Registry
                   inputs:
@@ -83,11 +88,18 @@ stages:
                         labels:
                           app: giga-data-sharing
                       stringData:
-                        DELTA_BEARER_TOKEN: $(deltaBearerToken)
-                        SAS_TOKEN: $(sasToken)
-                        STORAGE_ACCESS_KEY: $(storageAccessKey)
-                        STORAGE_ACCOUNT_NAME: $(storageAccountName)
-                        CONTAINER_NAME: $(storageContainerName)
+                        DELTA_BEARER_TOKEN: "$(deltaBearerToken)"
+                        AZURE_SAS_TOKEN: "$(sasToken)"
+                        STORAGE_ACCESS_KEY: "$(storageAccessKey)"
+                        STORAGE_ACCOUNT_NAME: "$(storageAccountName)"
+                        CONTAINER_NAME: "$(storageContainerName)"
+                        POSTGRESQL_USERNAME: "giga-data-sharing"
+                        POSTGRESQL_PASSWORD: "$(postgresqlPassword)"
+                        POSTGRESQL_DATABASE: "giga-data-sharing"
+                        ADMIN_API_KEY: $(adminApiKey)
+                        ADMIN_API_SECRET: $(adminApiSecret)
+                        APPLICATION_SECRET_KEY: $(secretKey)
+                        APP_DOMAIN: $(appDomain)
 
                 - task: HelmDeploy@0
                   displayName: Helm deploy Data Sharing Delta

custom-sas-provider/.dockerignore

@@ -0,0 +1,44 @@
+### Java template
+# Compiled class file
+*.class
+
+# Log file
+*.log
+
+# BlueJ files
+*.ctxt
+
+# Mobile Tools for Java (J2ME)
+.mtj.tmp/
+
+# Package Files #
+*.jar
+*.war
+*.nar
+*.ear
+*.zip
+*.tar.gz
+*.rar
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*
+replay_pid*
+
+### Maven template
+target/
+pom.xml.tag
+pom.xml.releaseBackup
+pom.xml.versionsBackup
+pom.xml.next
+release.properties
+dependency-reduced-pom.xml
+buildNumber.properties
+.mvn/timing.properties
+# https://github.com/takari/maven-wrapper#usage-without-binary-jar
+.mvn/wrapper/maven-wrapper.jar
+
+# Eclipse m2e generated files
+# Eclipse Core
+.project
+# JDT-specific (Eclipse Java Development Tools)
+.classpath

custom-sas-provider/.gitignore

@@ -0,0 +1,44 @@
+### Java template
+# Compiled class file
+*.class
+
+# Log file
+*.log
+
+# BlueJ files
+*.ctxt
+
+# Mobile Tools for Java (J2ME)
+.mtj.tmp/
+
+# Package Files #
+*.jar
+*.war
+*.nar
+*.ear
+*.zip
+*.tar.gz
+*.rar
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*
+replay_pid*
+
+### Maven template
+target/
+pom.xml.tag
+pom.xml.releaseBackup
+pom.xml.versionsBackup
+pom.xml.next
+release.properties
+dependency-reduced-pom.xml
+buildNumber.properties
+.mvn/timing.properties
+# https://github.com/takari/maven-wrapper#usage-without-binary-jar
+.mvn/wrapper/maven-wrapper.jar
+
+# Eclipse m2e generated files
+# Eclipse Core
+.project
+# JDT-specific (Eclipse Java Development Tools)
+.classpath