Merge pull request #251 from EHDEN/dev

Update master - release 1.3.0

.github/workflows/code_analysis.yml

@@ -86,5 +86,4 @@ jobs:
         pip install --upgrade pip
         pip install -r requirements-dev/requirements-prospector.txt -r dashboard_viewer/requirements.txt
     - name: prospector
-      run: |
-        prospector dashboard_viewer
+      run: DJANGO_SETTINGS_MODULE=dashboard_viewer.settings prospector dashboard_viewer

.github/workflows/tests.yml

@@ -39,4 +39,4 @@ jobs:
         export $(grep -v '^#' tests/.env | xargs -d '\n')
         cd dashboard_viewer
         python manage.py test --exclude-tag third-party-app
-        SINGLE_APPLICATION_MODE=n MAIN_APPLICATION_HOST=mainapp.host.com python manage.py test --tag third-party-app
+        #SINGLE_APPLICATION_MODE=n MAIN_APPLICATION_HOST=mainapp.host.com python manage.py test --tag third-party-app

.gitignore

@@ -1,15 +1,15 @@
 # documentation temporary files
 docs/src/_book
 
-# dashboard_viewer django app ERRORs logs
-logs
-
 # custom file created at docker/superset
 docker-init.sh
 
 # Docker volumes
 volumes/
 
+# backups configurations
+backups.conf
+
 # Retrived from https://github.com/github/gitignore/blob/21419e391a78f2487340b3b1240b988aaf15b54f/Node.gitignore
 # Logs
 logs

backups/backup.sh

@@ -8,9 +8,9 @@ echo_step() {
 
 set -e
 
-. $HOME/.dashboards_backups.conf
+. "$(dirname "$0")/backups.conf"
 
-if [ $RUN -eq 0 ] ; then
+if [ "$RUN" -eq 0 ] ; then
     echo "run was 0. exitting"
     exit 0
 fi
@@ -19,13 +19,13 @@ echo_step "1" "Create temporary directory"
 BACKUP_DIRECTORY_NAME=dashboards_backups_$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 40)
 TMP_BACKUP_DIRECTORY=$TMP_DIRECTORY/$BACKUP_DIRECTORY_NAME
 
-mkdir $TMP_BACKUP_DIRECTORY
+mkdir "$TMP_BACKUP_DIRECTORY"
 EXIT_STATUS=0
 
 (
     echo_step "2" "Get into the docker directory"
     PREVIOUS_PWD=$(pwd)
-    cd $(dirname "$0")
+    cd "$(dirname "$0")"
     (
         cd ../docker
 
@@ -45,7 +45,7 @@ EXIT_STATUS=0
 
         REDIS_CONTAINER_ID=$(docker-compose ps -q redis)
 
-        docker cp -a $REDIS_CONTAINER_ID:/data/dump.rdb $TMP_BACKUP_DIRECTORY/redis.rdb
+        docker cp -a $REDIS_CONTAINER_ID:/data/dump.rdb "$TMP_BACKUP_DIRECTORY/redis.rdb"
 
         echo_step "5" "Extract Dashboards's media files"
         MEDIA_ROOT=$(docker-compose exec -T dashboard sh -c """
@@ -63,28 +63,28 @@ print(settings.MEDIA_ROOT, end=\"\")
 
         # copy media files to backup folder
         DASHBOARDS_CONTAINER_ID=$(docker-compose ps -q dashboard)
-        docker cp -a $DASHBOARDS_CONTAINER_ID:$MEDIA_ROOT $TMP_BACKUP_DIRECTORY
+        docker cp -a $DASHBOARDS_CONTAINER_ID:$MEDIA_ROOT "$TMP_BACKUP_DIRECTORY"
 
         echo_step "6" "Compress gathered data"
         COMPRESSED_FILE_PATH=$TMP_DIRECTORY/${APP_NAME}_$(date +"%Y%m%d%H%M%S").zip
         (
-            cd $TMP_DIRECTORY
-            zip -q -r $COMPRESSED_FILE_PATH $BACKUP_DIRECTORY_NAME
-            #tar -C $TMP_DIRECTORY -cJf $COMPRESSED_FILE_PATH $BACKUP_DIRECTORY_NAME
+            cd "$TMP_DIRECTORY"
+            zip -q -r "$COMPRESSED_FILE_PATH" $BACKUP_DIRECTORY_NAME
+            #tar -C "$TMP_DIRECTORY" -cJf "$COMPRESSED_FILE_PATH" $BACKUP_DIRECTORY_NAME
 
             echo_step "7" "Send to $SERVER"
-            backup_uploader $APP_NAME $SERVER $CREDENTIALS_FILE_PATH $BACKUP_CHAIN_CONFIG $COMPRESSED_FILE_PATH
+            backup_uploader "$APP_NAME" "$SERVER" "$CREDENTIALS_FILE_PATH" "$BACKUP_CHAIN_CONFIG" "$COMPRESSED_FILE_PATH"
         ) || EXIT_STATUS=$?
-        rm -f $COMPRESSED_FILE_PATH
+        rm -f "$COMPRESSED_FILE_PATH"
 
         exit $EXIT_STATUS
     ) || EXIT_STATUS=$?
-    cd $PREVIOUS_PWD
+    cd "$PREVIOUS_PWD"
 
     exit $EXIT_STATUS
 ) || EXIT_STATUS=$?
 
-rm -rf $TMP_BACKUP_DIRECTORY
+rm -rf "$TMP_BACKUP_DIRECTORY"
 
 if [ $EXIT_STATUS -ne 0 ] ; then
     echo "Failed with exit code $EXIT_STATUS"

backups/restore.sh

@@ -0,0 +1,96 @@
+#!/bin/sh
+
+set -e
+
+if [ $# -ne 1 ] ; then
+  1>&2 echo "Usage: restore.sh [zip backup file]"
+  exit 1
+fi
+
+STEP_COUNT=5
+
+echo_step() {
+  printf "%3s/%s %s\n" "$1" "$STEP_COUNT" "$2"
+}
+
+. "$(dirname "$0")/backups.conf"
+
+echo_step 1 "Extracting backup file"
+
+BACKUP_ROOT_DIRECTORY=$(unzip -qql "$1" | head -n1 | sed -r '1 {s/([ ]+[^ ]+){3}\s+//;q}')
+unzip "$1" -d "$TMP_DIRECTORY" >/dev/null
+BACKUP_ROOT_DIRECTORY="$TMP_DIRECTORY/$BACKUP_ROOT_DIRECTORY"
+
+EXIT_STATUS=0
+
+(
+  echo_step 2 "Get into the docker directory"
+  PREVIOUS_PWD=$(pwd)
+  cd "$(dirname "$0")"
+  (
+    cd ../docker
+
+    echo_step 3 "Restoring postgres"
+    docker-compose stop dashboard dashboard_worker superset superset-worker superset-worker-beat >/dev/null 2>&1
+    docker-compose up -d postgres >/dev/null 2>&1
+
+    until docker-compose exec -T postgres sh -c "pg_isready" >/dev/null 2>&1 ; do
+        sleep 2
+    done
+
+    CONTAINER_ID=$(docker-compose ps -q postgres)
+    docker cp "$BACKUP_ROOT_DIRECTORY/postgres.sql" $CONTAINER_ID:/tmp
+    docker-compose exec -T postgres sh -c """
+psql -f /tmp/postgres.sql -U \$POSTGRES_USER -d postgres
+rm /tmp/postgres.sql
+"""
+
+    echo_step 4 "Restoring redis"
+    docker-compose up -d redis >/dev/null 2>&1
+    CONTAINER_ID=$(docker-compose ps -q redis)
+    docker cp "$BACKUP_ROOT_DIRECTORY/redis.rdb" $CONTAINER_ID:/
+    docker-compose exec -T redis sh -c """
+mv /redis.rdb /data
+"""
+    docker-compose restart redis >/dev/null 2>&1
+
+    echo_step 5 "Restoring Dashboards' media files"
+    docker-compose up -d dashboard >/dev/null 2>&1
+    MEDIA_ROOT=$(docker-compose exec -T dashboard sh -c """
+echo '''from django.conf import settings
+print(settings.MEDIA_ROOT, end=\"\")
+''' | python manage.py shell 2>/dev/null""")
+
+    # fix MEDIA_ROOT if a relative path is returned
+    case $MEDIA_ROOT in
+        "/"*) ;;
+        *)
+            MEDIA_ROOT="/app/$MEDIA_ROOT"
+        ;;
+    esac
+
+    CONTAINER_ID=$(docker-compose ps -q dashboard)
+
+    docker-compose exec -T dashboard sh -c """
+cd $MEDIA_ROOT
+find . -mindepth 1 -maxdepth 1 -exec rm -r {} +
+"""
+    docker cp -a "$BACKUP_ROOT_DIRECTORY/$(basename "$MEDIA_ROOT")" $CONTAINER_ID:"$MEDIA_ROOT"
+    docker-compose exec -T dashboard sh -c """
+cd $MEDIA_ROOT
+find $(basename "$MEDIA_ROOT") -mindepth 1 -maxdepth 1 -exec mv -t . -- {} +
+rmdir $(basename "$MEDIA_ROOT")
+"""
+
+    docker-compose up -d
+  ) || EXIT_STATUS=$?
+  cd "$PREVIOUS_PWD"
+
+  exit $EXIT_STATUS
+) || EXIT_STATUS=$?
+rm -rf "$BACKUP_ROOT_DIRECTORY"
+
+if [ $EXIT_STATUS -ne 0 ] ;then
+    echo "Failed with exit code $EXIT_STATUS"
+    exit $EXIT_STATUS
+fi

dashboard_viewer/dashboard_viewer/settings.py

@@ -9,8 +9,9 @@
 For the full list of settings and their values, see
 https://docs.djangoproject.com/en/2.2/ref/settings/
 """
-
+import logging
 import os
+from collections import OrderedDict
 from distutils.util import strtobool
 
 from constance.signals import config_updated
@@ -24,43 +25,43 @@
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/
 
-# SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = os.environ["SECRET_KEY"]
-
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = os.environ.get("DASHBOARD_VIEWER_ENV", "development") == "development"
 
 
-_LOGS_DIR = os.path.join(BASE_DIR, "logs")
-if not os.path.exists(_LOGS_DIR):
-    os.makedirs(_LOGS_DIR, exist_ok=True)
-elif not os.path.isdir(_LOGS_DIR):
-    raise TypeError('file "logs" is not a directory.')
-
 LOGGING = {
     "version": 1,
-    "filters": {
-        "require_debug_true": {
-            "()": "django.utils.log.RequireDebugTrue",
+    "disable_existing_loggers": False,
+    "formatters": {
+        "custom": {
+            "format": "%(asctime)s %(levelname)s %(name)s:%(lineno)s %(message)s",
         },
     },
     "handlers": {
-        "file": {
-            "level": "ERROR",
-            "filters": ["require_debug_true"],
-            "class": "logging.FileHandler",
-            "filename": os.path.join(_LOGS_DIR, "errors.log"),
-        }
+        "console": {
+            "class": "logging.StreamHandler",
+            "formatter": "custom",
+        },
     },
     "loggers": {
-        "django": {
-            "handlers": ["file"],
-            "level": "ERROR",
-            "propagate": True,
+        "root": {
+            "handlers": ["console"],
+            "level": "DEBUG" if DEBUG else "INFO",
         },
     },
 }
 
+
+_DEFAULT_SECRET_KEY = "CHANGE_ME"  # noqa
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = os.environ.get("SECRET_KEY", _DEFAULT_SECRET_KEY)
+if not DEBUG and SECRET_KEY == _DEFAULT_SECRET_KEY:
+    logging.getLogger(__name__).warning(
+        "Using the default secret key. If this is a production environment please change it.",
+    )
+
+
 ALLOWED_HOSTS = ["*"]
 
 
@@ -275,6 +276,21 @@
         "If a Data Source owner can change the draft status when editing its details",
         bool,
     ),
+    "SUPERSET_HOST": (
+        "https://superset.ehden.eu",
+        "Host of the target superset installation. Used to redirect to the dashboard of a database",
+        str,
+    ),
+    "DATABASE_DASHBOARD_IDENTIFIER": (
+        "database-level-dashboard",
+        "Identifier of the database dashboard on the Superset installation.",
+        str,
+    ),
+    "DATABASE_FILTER_ID": (
+        69,
+        "Id of the database filter present in the Database Dashboard",
+        int,
+    ),
     "TABS_LOGO_CONTAINER_CSS": (
         "padding: 5px 5px 5px 5px;\nheight: 100px;\nmargin-bottom: 10px;\n",
         "Css for the div container of the logo image",
@@ -295,6 +311,26 @@
     ),
 }
 
+CONSTANCE_CONFIG_FIELDSETS = OrderedDict(
+    [
+        ("Application Attributes", ("APP_LOGO_IMAGE", "APP_LOGO_URL", "APP_TITLE")),
+        (
+            "Uploader Texts",
+            (
+                "UPLOADER_EXECUTE_EXPORT_PACKAGE",
+                "UPLOADER_UPLOAD",
+                "UPLOADER_AUTO_UPDATE",
+            ),
+        ),
+        ("Uploader Settings", ("UPLOADER_ALLOW_EDIT_DRAFT_STATUS",)),
+        (
+            "Superset",
+            ("SUPERSET_HOST", "DATABASE_DASHBOARD_IDENTIFIER", "DATABASE_FILTER_ID"),
+        ),
+        ("Tabs (Deprecated)", ("TABS_LOGO_CONTAINER_CSS", "TABS_LOGO_IMG_CSS")),
+    ]
+)
+
 
 @receiver(config_updated)
 def constance_updated(key, old_value, **_):

dashboard_viewer/dashboard_viewer/urls.py

@@ -39,7 +39,7 @@
     path("martor/", include("martor.urls")),
     path("uploader/", include("uploader.urls")),
     re_path(
-        fr'^{re.escape(settings.MEDIA_URL.lstrip("/"))}(?P<path>.*)$',
+        rf'^{re.escape(settings.MEDIA_URL.lstrip("/"))}(?P<path>.*)$',
         serve,
         kwargs={"document_root": settings.MEDIA_ROOT},
     )

dashboard_viewer/docker-entrypoint.sh

@@ -9,7 +9,7 @@ python manage.py compilescss
 python manage.py collectstatic --noinput --ignore="*.scss"
 
 if [ "${DASHBOARD_VIEWER_ENV}" = "production" ]; then
-    exec gunicorn dashboard_viewer.wsgi:application --bind 0.0.0.0:8000 --workers 5
+    exec gunicorn dashboard_viewer.wsgi:application --bind 0.0.0.0:8000 --workers 1 --worker-class gthread --thread 16
 else
     python manage.py runserver 0.0.0.0:8000
 fi

dashboard_viewer/materialized_queries_manager/management/commands/refresh_mat_views.py

@@ -2,6 +2,7 @@
 
 from django.core.cache import cache
 from django.core.management.base import BaseCommand
+
 from materialized_queries_manager.utils import refresh
 
 logger = logging.getLogger(__name__)

dashboard_viewer/materialized_queries_manager/tasks.py

@@ -9,6 +9,7 @@
 from django.core import serializers
 from django.core.cache import cache
 from django.db import connections, ProgrammingError, router, transaction
+
 from materialized_queries_manager.models import MaterializedQuery
 from materialized_queries_manager.utils import refresh
 

dashboard_viewer/materialized_queries_manager/utils.py

@@ -1,8 +1,9 @@
 from django.core.cache import cache
 from django.db import connections
-from materialized_queries_manager.models import MaterializedQuery
 from redis_rw_lock import RWLock
 
+from materialized_queries_manager.models import MaterializedQuery
+
 
 def refresh(logger, db_id=None, query_set=None):
     # Only one worker can update the materialized views at the same time -> same as -> only one thread

dashboard_viewer/requirements.in

@@ -8,13 +8,13 @@ django-markdownify==0.9.0                 # markdown to html
 django-model-utils==4.2.0                 # get specific type of subclasses after requesting buttons on tabsManager app
 django-sass-processor==1.1                # automate scss devolopment
 django-redis==5.1.0                       # acess redis through a programmatic API
-django==3.2.10
+django==3.2.13
 djangorestframework==3.13.1               # expose tabs content through an API
 libsass==0.21.0                           # to compile scss files into css
 gunicorn==20.1.0                          # for production deployment
 martor==1.6.7                             # markdown editor in admin app
 pandas==1.3.5                             # to handle achilles results files and their data
-Pillow==8.4.0                             # image fields (App Logo)
+Pillow==9.0.1                             # image fields (App Logo)
 psycopg2-binary==2.9.2                    # communicate with postgres
 redis==3.5.3                              # comunicate with redis (celery)
 git+https://github.com/bioinformatics-ua/redis-rw-lock.git#egg=redis-rw-lock