Fix/1140 status db indexing (main) (#1144)

* apply index opts

* Add status index_DB migration file

* Add script to test DB migrations

Add basic CI check for DB update miration

fetch all commits to parse out last release

Fix DB create script

adjust permissions

limit py env to server requ

* Updated Package Requirements: waitress==3.0.1

* Fix CVE-2023-43804

---------

Co-authored-by: awsbuild <[email protected]>

Author: sambles

.github/workflows/test-db-migration.yml

@@ -0,0 +1,62 @@
+name: DB migration Test
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      prev_release_tag:
+        description: 'OasisPlatform tag [semvar]'
+        required: false
+
+  workflow_call:
+    inputs:
+      prev_release_tag:
+        description: 'Platform tag to set the DB migration from'
+        required: false
+        type: string
+      platform_branch:
+        description: "Platform branch to migrate to (remote trigger) [git ref]"
+        required: True
+        type: string
+
+jobs:
+  DB_migrate:
+    runs-on: ubuntu-22.04
+    env:
+      PLAT_BRANCH: ${{ github.ref }}
+      PREV_RELEASE_TAG: ${{ inputs.prev_release_tag }}
+
+    steps:
+    - name: Branch selection (remote trigger)
+      if: inputs.platform_branch != ''
+      run: echo "PLAT_BRANCH=${{ inputs.platform_branch }}" >> $GITHUB_ENV
+
+    - name: Checkout
+      uses: actions/checkout@v3
+      with:
+        repository: OasisLMF/OasisPlatform
+        ref: ${{ env.PLAT_BRANCH }}
+        fetch-depth: 0
+
+    - name: Find 'prev_release_tag'
+      if: inputs.prev_release_tag == ''
+      run: |
+        tag=$( ./scripts/find_release.sh -t 1)
+        echo "PREV_RELEASE_TAG=$tag" >> $GITHUB_ENV
+
+    - name: Create DB to migrate from
+      run: |
+        ./scripts/create-db-sqlite3.sh ${{ env.PREV_RELEASE_TAG }}
+
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.12'
+    - run: pip install -r requirements-server.txt
+
+    - name: Run Migration
+      run: |
+        python ./manage.py migrate

kubernetes/worker-controller/requirements.txt

@@ -38,7 +38,7 @@ six==1.16.0
     # via
     #   kubernetes-asyncio
     #   python-dateutil
-urllib3==1.26.12
+urllib3==2.2.3
     # via kubernetes-asyncio
 websockets==10.3
     # via -r kubernetes/worker-controller/requirements.in

requirements.txt

@@ -610,7 +610,7 @@ vine==5.1.0
     #   kombu
 virtualenv==20.27.0
     # via tox
-waitress==3.0.0
+waitress==3.0.1
     # via webtest
 wcwidth==0.2.13
     # via prompt-toolkit

scripts/create-db-sqlite3.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -e 
+function usage {
+    echo "Usage: $0 <OasisPlatform Tag>,  e.g. 2.3.10, or 1.28.9"
+    echo
+    echo " This script initializes an db.sqlite3 DB file, used for testing migration changes added to the platform."
+}
+
+if [ -z "$1" ]; then
+    usage
+    exit 1
+else
+    SERVER_TAG=$1
+fi
+
+SERVER_IMAGE='coreoasis/api_server'
+SQL_FILE='src/server/oasisapi/db.sqlite3'
+ENTRY='/usr/bin/python3'
+CMD='manage.py migrate'
+#CMD='/bin/bash'
+
+if [ -f $SQL_FILE ]; then
+    echo "Found existing file $SQL_FILE - deleting and creating a empty file"
+    rm -f $SQL_FILE
+fi
+touch $SQL_FILE
+chmod 777 $SQL_FILE
+
+
+echo "Creating a blank db.sqlite3 file for OasisPlatform $SERVER_TAG"
+echo 
+docker run -e OASIS_SERVER_DB_ENGINE=django.db.backends.sqlite3 --entrypoint $ENTRY -v ./$SQL_FILE:/var/www/oasis/$SQL_FILE $SERVER_IMAGE:$SERVER_TAG $CMD
+echo "-- Done --"

src/server/oasisapi/analyses/migrations/0015_auto_20241125_1407.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.25 on 2024-11-25 14:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('analyses', '0014_merge_20240202_1452'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='analysis',
+            name='status',
+            field=models.CharField(choices=[('NEW', 'New'), ('INPUTS_GENERATION_ERROR', 'Inputs generation error'), ('INPUTS_GENERATION_CANCELLED', 'Inputs generation cancelled'), ('INPUTS_GENERATION_STARTED', 'Inputs generation started'), ('INPUTS_GENERATION_QUEUED', 'Inputs generation added to queue'), ('READY', 'Ready'), ('RUN_QUEUED', 'Run added to queue'), ('RUN_STARTED', 'Run started'), ('RUN_COMPLETED', 'Run completed'), ('RUN_CANCELLED', 'Run cancelled'), ('RUN_ERROR', 'Run error')], db_index=True, default='NEW', editable=False, max_length=27),
+        ),
+        migrations.AlterField(
+            model_name='analysistaskstatus',
+            name='status',
+            field=models.CharField(choices=[('PENDING', 'Task waiting to be added to the queue'), ('QUEUED', 'Task added to queue'), ('STARTED', 'Task started'), ('COMPLETED', 'Task completed'), ('CANCELLED', 'Task cancelled'), ('ERROR', 'Task error')], db_index=True, default='PENDING', editable=False, max_length=9),
+        ),
+    ]

src/server/oasisapi/analyses/models.py

@@ -94,6 +94,7 @@ class AnalysisTaskStatus(models.Model):
         choices=status_choices,
         default=status_choices.PENDING,
         editable=False,
+        db_index=True,
     )
     pending_time = models.DateTimeField(null=True, auto_now_add=True, editable=False)
     queue_time = models.DateTimeField(null=True, default=None, editable=False)
@@ -166,7 +167,7 @@ class Analysis(TimeStampedModel):
     model = models.ForeignKey(AnalysisModel, on_delete=models.CASCADE, related_name='analyses', help_text=_('The model to link the analysis to'))
     name = models.CharField(help_text='The name of the analysis', max_length=255)
     status = models.CharField(max_length=max(len(c) for c in status_choices._db_values),
-                              choices=status_choices, default=status_choices.NEW, editable=False)
+                              choices=status_choices, default=status_choices.NEW, editable=False, db_index=True)
     run_mode = models.CharField(max_length=max(len(c) for c in run_mode_choices._db_values),
                                 choices=run_mode_choices, default=None, editable=False, null=True)
     task_started = models.DateTimeField(editable=False, null=True, default=None)