Fix/1140 status db indexing (stable 2.3.x) (#1145)

* test merge from main

* copy in missing migration file (from main)

* Add script to test DB migrations

Add basic CI check for DB update miration

fetch all commits to parse out last release

Fix DB create script

adjust permissions

limit py env to server requ

* Add missing migration file

* Fix CVE-2023-43804

.github/workflows/test-db-migration.yml

@@ -0,0 +1,62 @@
+name: DB migration Test
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      prev_release_tag:
+        description: 'OasisPlatform tag [semvar]'
+        required: false
+
+  workflow_call:
+    inputs:
+      prev_release_tag:
+        description: 'Platform tag to set the DB migration from'
+        required: false
+        type: string
+      platform_branch:
+        description: "Platform branch to migrate to (remote trigger) [git ref]"
+        required: True
+        type: string
+
+jobs:
+  DB_migrate:
+    runs-on: ubuntu-22.04
+    env:
+      PLAT_BRANCH: ${{ github.ref }}
+      PREV_RELEASE_TAG: ${{ inputs.prev_release_tag }}
+
+    steps:
+    - name: Branch selection (remote trigger)
+      if: inputs.platform_branch != ''
+      run: echo "PLAT_BRANCH=${{ inputs.platform_branch }}" >> $GITHUB_ENV
+
+    - name: Checkout
+      uses: actions/checkout@v3
+      with:
+        repository: OasisLMF/OasisPlatform
+        ref: ${{ env.PLAT_BRANCH }}
+        fetch-depth: 0
+
+    - name: Find 'prev_release_tag'
+      if: inputs.prev_release_tag == ''
+      run: |
+        tag=$( ./scripts/find_release.sh -t 1)
+        echo "PREV_RELEASE_TAG=$tag" >> $GITHUB_ENV
+
+    - name: Create DB to migrate from
+      run: |
+        ./scripts/create-db-sqlite3.sh ${{ env.PREV_RELEASE_TAG }}
+
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.12'
+    - run: pip install -r requirements-server.txt
+
+    - name: Run Migration
+      run: |
+        python ./manage.py migrate

scripts/create-db-sqlite3.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -e 
+function usage {
+    echo "Usage: $0 <OasisPlatform Tag>,  e.g. 2.3.10, or 1.28.9"
+    echo
+    echo " This script initializes an db.sqlite3 DB file, used for testing migration changes added to the platform."
+}
+
+if [ -z "$1" ]; then
+    usage
+    exit 1
+else
+    SERVER_TAG=$1
+fi
+
+SERVER_IMAGE='coreoasis/api_server'
+SQL_FILE='src/server/oasisapi/db.sqlite3'
+ENTRY='/usr/bin/python3'
+CMD='manage.py migrate'
+#CMD='/bin/bash'
+
+if [ -f $SQL_FILE ]; then
+    echo "Found existing file $SQL_FILE - deleting and creating a empty file"
+    rm -f $SQL_FILE
+fi
+touch $SQL_FILE
+chmod 777 $SQL_FILE
+
+
+echo "Creating a blank db.sqlite3 file for OasisPlatform $SERVER_TAG"
+echo 
+docker run -e OASIS_SERVER_DB_ENGINE=django.db.backends.sqlite3 --entrypoint $ENTRY -v ./$SQL_FILE:/var/www/oasis/$SQL_FILE $SERVER_IMAGE:$SERVER_TAG $CMD
+echo "-- Done --"

src/server/oasisapi/analyses/migrations/0012_analysis_raw_output_files.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.19 on 2023-07-18 09:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('files', '0005_relatedfile_oed_validated'),
+        ('analyses', '0011_auto_20230724_1134'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='analysis',
+            name='raw_output_files',
+            field=models.ManyToManyField(blank=True, related_name='raw_output_files_analyses', to='files.RelatedFile'),
+        ),
+    ]

src/server/oasisapi/analyses/migrations/0014_merge_20240202_1452.py

@@ -0,0 +1,14 @@
+# Generated by Django 3.2.19 on 2024-02-02 14:52
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('analyses', '0012_analysis_raw_output_files'),
+        ('analyses', '0013_analysis_chunking_options'),
+    ]
+
+    operations = [
+    ]

src/server/oasisapi/analyses/migrations/0015_auto_20241125_1407.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.25 on 2024-11-25 14:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('analyses', '0014_merge_20240202_1452'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='analysis',
+            name='status',
+            field=models.CharField(choices=[('NEW', 'New'), ('INPUTS_GENERATION_ERROR', 'Inputs generation error'), ('INPUTS_GENERATION_CANCELLED', 'Inputs generation cancelled'), ('INPUTS_GENERATION_STARTED', 'Inputs generation started'), ('INPUTS_GENERATION_QUEUED', 'Inputs generation added to queue'), ('READY', 'Ready'), ('RUN_QUEUED', 'Run added to queue'), ('RUN_STARTED', 'Run started'), ('RUN_COMPLETED', 'Run completed'), ('RUN_CANCELLED', 'Run cancelled'), ('RUN_ERROR', 'Run error')], db_index=True, default='NEW', editable=False, max_length=27),
+        ),
+        migrations.AlterField(
+            model_name='analysistaskstatus',
+            name='status',
+            field=models.CharField(choices=[('PENDING', 'Task waiting to be added to the queue'), ('QUEUED', 'Task added to queue'), ('STARTED', 'Task started'), ('COMPLETED', 'Task completed'), ('CANCELLED', 'Task cancelled'), ('ERROR', 'Task error')], db_index=True, default='PENDING', editable=False, max_length=9),
+        ),
+    ]

src/server/oasisapi/analyses/models.py

@@ -94,6 +94,7 @@ class AnalysisTaskStatus(models.Model):
         choices=status_choices,
         default=status_choices.PENDING,
         editable=False,
+        db_index=True,
     )
     pending_time = models.DateTimeField(null=True, auto_now_add=True, editable=False)
     queue_time = models.DateTimeField(null=True, default=None, editable=False)
@@ -166,7 +167,7 @@ class Analysis(TimeStampedModel):
     model = models.ForeignKey(AnalysisModel, on_delete=models.CASCADE, related_name='analyses', help_text=_('The model to link the analysis to'))
     name = models.CharField(help_text='The name of the analysis', max_length=255)
     status = models.CharField(max_length=max(len(c) for c in status_choices._db_values),
-                              choices=status_choices, default=status_choices.NEW, editable=False)
+                              choices=status_choices, default=status_choices.NEW, editable=False, db_index=True)
     run_mode = models.CharField(max_length=max(len(c) for c in run_mode_choices._db_values),
                                 choices=run_mode_choices, default=None, editable=False, null=True)
     task_started = models.DateTimeField(editable=False, null=True, default=None)