Imp/Merge existing pull requests

.github/workflows/code_analysis.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        pythonversion: [3.7]
+        pythonversion: [3.9]
 
     steps:
     - uses: actions/checkout@v2
@@ -38,7 +38,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        pythonversion: [3.7]
+        pythonversion: [3.9]
 
     steps:
     - uses: actions/checkout@v2
@@ -66,7 +66,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        pythonversion: [3.7]
+        pythonversion: [3.9]
 
     steps:
     - uses: actions/checkout@v2
@@ -88,4 +88,3 @@ jobs:
     - name: prospector
       run: |
         prospector dashboard_viewer
-        prospector docker/superset

.github/workflows/publish_docker_images.yml

@@ -0,0 +1,62 @@
+name: Build & Publish Docker Images
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  dashboards:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+
+      - name: Extract metadata
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: aspedrosa/networkdashboards
+          tags: |
+              type=pep440,pattern={{version}}
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v2
+        with:
+          context: dashboard_viewer
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+
+  superset:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - name: Extract metadata
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: aspedrosa/networkdashboards_superset
+          tags: |
+              type=pep440,pattern={{version}}
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v2
+        with:
+          context: superset
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}

.github/workflows/test_docker_images.yml

@@ -0,0 +1,50 @@
+name: Test Build Docker Images
+
+on:
+  pull_request:
+    branches: [ dev ]
+
+jobs:
+  # JOB to run change detection
+  changes:
+    runs-on: ubuntu-latest
+    # Set job outputs to values from filter step
+    outputs:
+      dashboards: ${{ steps.filter.outputs.dashboards }}
+      superset: ${{ steps.filter.outputs.superset }}
+    steps:
+    # For pull requests it's not necessary to checkout the code
+    - uses: dorny/paths-filter@v2
+      id: filter
+      with:
+        filters: |
+          dashboards:
+            - 'dashboard_viewer/**'
+          superset:
+            - 'superset/**'
+
+  dashboards:
+    needs: changes
+    if: ${{ needs.changes.outputs.dashboards == 'true' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+
+      - name: Test build
+        run: |
+          docker build dashboard_viewer
+
+  superset:
+    needs: changes
+    if: ${{ needs.changes.outputs.superset == 'true' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - name: Test build
+        run: |
+          docker build superset

.github/workflows/tests.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        pythonversion: [3.7]
+        pythonversion: [3.9]
 
     steps:
     - uses: actions/checkout@v2
@@ -27,10 +27,8 @@ jobs:
           ${{ runner.os }}-pip-
     - name: Setup Data Containers
       run: |
-        export $(grep -v '^#' tests/.env | xargs -d '\n')
-        cp tests/docker-compose.yml docker
-        cp tests/init-dbs.sh docker/postgres-entrypoint
-        cd docker
+        cd tests
+        export $(grep -v '^#' .env | xargs -d '\n')
         docker-compose up -d
     - name: Install Dependencies
       run: |
@@ -40,5 +38,5 @@ jobs:
       run: |
         export $(grep -v '^#' tests/.env | xargs -d '\n')
         cd dashboard_viewer
-        python manage.py migrate
-        python manage.py test
+        python manage.py test --exclude-tag third-party-app
+        SINGLE_APPLICATION_MODE=n MAIN_APPLICATION_HOST=mainapp.host.com python manage.py test --tag third-party-app

.gitignore

@@ -1,3 +1,6 @@
+# documentation temporary files
+docs/src/_book
+
 # dashboard_viewer django app ERRORs logs
 logs
 

.pylintrc

@@ -0,0 +1,111 @@
+[MESSAGES CONTROL]
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once). You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use "--disable=all --enable=classes
+# --disable=W".
+disable=unnecessary-semicolon,
+        superfluous-parens,
+        bad-mcs-classmethod-argument,
+        too-many-lines,
+        bad-whitespace,
+        I0014,
+        relative-import,
+        E1103,
+        C0322,
+        fixme,
+        too-many-instance-attributes,
+        no-init,
+        C0323,
+        trailing-whitespace,
+        property-on-old-class,
+        too-few-public-methods,
+        invalid-name,
+        exec-used,
+        assignment-from-none,
+        C0324,
+        abstract-class-little-used,
+        pointless-except,
+        redefined-outer-name,
+        star-args,
+        missing-final-newline,
+        bad-mcs-method-argument,
+        empty-docstring,
+        no-name-in-module,
+        global-variable-undefined,
+        too-many-return-statements,
+        W0701,
+        bad-builtin,
+        W5103,
+        deprecated-lambda,
+        abstract-method,
+        no-member,
+        unused-format-string-key,
+        no-self-use,
+        wildcard-import,
+        too-many-public-methods,
+        broad-except,
+        bad-classmethod-argument,
+        bad-continuation,
+        attribute-defined-outside-init,
+        W0713,
+        ungrouped-imports,
+        anomalous-backslash-in-string,
+        too-many-ancestors,
+        wrong-import-order,
+        multiple-statements,
+        missing-docstring,
+        no-self-argument,
+        global-statement,
+        super-init-not-called,
+        raise-missing-from,
+        protected-access,
+        invalid-str-returned,
+        invalid-repr-returned,
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+enable=
+
+
+[FORMAT]
+
+# Maximum number of characters on a single line.
+max-line-length=159
+
+# Maximum number of lines in a module.
+max-module-lines=1000
+
+
+[DESIGN]
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
+
+# Maximum number of branch for function / method body.
+max-branches=15
+
+# Maximum number of locals for function / method body.
+max-locals=15
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+# Maximum number of return / yield for function / method body.
+max-returns=6
+
+# Maximum number of statements in function / method body.
+max-statements=60
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=1

backups/README.md

@@ -1,100 +1,3 @@
 # Backup
 
-1. Create a credentials file (the structure of the file depends on the target cloud server)
-
-2. Create a `.dashboards_backups.conf` file under your home directory (variable `$HOME`) using `dashboards_backups.conf.example` as base, setting the appropriate value for the several variables.
-
-   For variables associated with files and directories always use *absolute* paths.
-
-   Variables:
-
-   - `RUN`: Set it to `0` if you don't want the next scheduled backup to run.
-
-     This variable allows you to cancel any backup runs while you are doing some maintenance on the application.
-
-   - `CONSTANCE_REDIS_DB`: Number of the Redis database where the django constance config is stored. The default value is 2. This value should be the same as the environment variable `REDIS_CONSTANCE_DB` of the dashboard container.
-
-   - The following variables are associated with the arguemtns of the `backup_uploader` python package. Check its [usage](https://github.com/aspedrosa/BackupUploader#usage) for more details:
-
-      - `APP_NAME`: The backup process will generate some directories with this name in places that are shared with other applications.
-
-      - `SERVER`: The name of the target cloud server to where backups should be uploaded (dropbox or mega).
-
-      - `BACKUP_CHAIN_CONFIG`: Allows having different directories with backups of different ages.
-
-      - `CREDENTIALS_FILE_PATH`: File containing the credentials to access the server to upload the backup file.
-
-3. Install the `backup_uploader` python package by following its [install](https://github.com/aspedrosa/BackupUploader#install) instructions.
-
-4. Schedule your backups
-
-   ```sh
-   *    *    *   *    *  Command_to_execute
-   |    |    |    |   |       
-   |    |    |    |    Day of the Week ( 0 - 6 ) ( Sunday = 0 )
-   |    |    |    |
-   |    |    |    Month ( 1 - 12 )
-   |    |    |
-   |    |    Day of Month ( 1 - 31 )
-   |    |
-   |    Hour ( 0 - 23 )
-   |
-   Min ( 0 - 59 ) 
-   ```
-
-   (Retrived from: [Tutorialspoint](https://www.tutorialspoint.com/unix_commands/crontab.htm))
-
-   Ex: To run every day at 3:00 am
-
-   1. `crontab -e`
-
-   2. Add entry `0 3 * * * $HOME/NetworkDashboards/backups/backup.sh` (The path to the backup script might be different)
-
-### Restore
-
-1. Select the compressed backup you want to restore and decompress it:
-
-   `tar -xJf BACKUP_FILE.tar.xz`.
-
-2. 1. **Redis**
-
-      1. Make sure the redis docker container is down.
-
-      2. (Re)place the file `dump.rdb` on the redis volume by the file `redis.rdb`. By default the redis volume is located where this repository was cloned on the directory `docker/volumes/redis`.
-
-      3. Change its permissions, owner and group:
-
-         ```shell
-         chmod 0644 docker/volumes/redis/dump.rdb
-         sudo chown -R 999:999 docker/volumes/redis
-         ```
-
-   2. **Postgres**
-
-      1. Make sure all containers that make changes on the database are stopped.
-
-      2. Copy the file `postgres_backup.sql` into the postgres container
-
-         `docker cp postgres.sql [CONTAINER_ID]:/tmp`.
-
-      5. Execute the backup script:
-
-         `docker exec -u root dashboard_viewer_postgres_1 psql -f /tmp/postgres_backup.sql -U \$POSTGRES_USER -d \$POSTGRES_DB`.
-
-   3. **Media Files** If you have a volume pointing to where the media files are stored, replace all files with the ones present on the downloaded backup file. Else:
-
-      1. Bring the dashoard container up `docker-compose up -d dashboard`
-
-      2. Enter in the container `docker exec -it [CONTAINER_ID] bash`
-
-      3. If you don't know where the media files are stored you can check the value of the MEDIA_ROOT variable
-
-         1. `python manage.py shell`
-
-         2. `from django.conf import settings`
-
-         3. `print(settings.MEDIA_ROOT)`
-
-      4. Remove the entire MEDIA_ROOT directory and exit the container
-
-      5. Copy the media directory present on the backup file to the catalogue container `docker cp -a collected-media [CONTAINER_ID]:[MEDIA_ROOT_PARENT_PATH]`
+Refer to the [Backups](https://ehden.github.io/NetworkDashboards/backups.html) chapter of the documentation