First Class Files (#856)

* added model changes and schema changes * updated some logic and added first test * fixed broken texts * working tests, functional * added migration script * fixed migration script with permissions * testing prod pipes, temp wfl change * switch to using list operations for output files * refactored to use prefixes when listing * Fixed blob checks in output creation * uncomment test gcs client * working migration and fcf tests * optimised client calls * review cleanup and refactoring * added appropriate var to mock correctly * added monkey patch for gcs client * removed default var * patching all tests for gcs client * Bump version: 7.3.3 → 7.4.0
populationgenomics · Aug 19, 2024 · cc0961d · cc0961d
1 parent 5a0e49f
commit cc0961d
Show file tree

Hide file tree

Showing 34 changed files with 1,403 additions and 70 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 7.3.3
+current_version = 7.4.0
 commit = True
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>[A-z0-9-]+)

diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
@@ -18,12 +18,12 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: '3.11'
 
       - uses: actions/setup-java@v4
         with:
-          distribution: "temurin" # See 'Supported distributions' for available options
-          java-version: "17"
+          distribution: 'temurin' # See 'Supported distributions' for available options
+          java-version: '17'
 
       - name: Setup build env
         run: |
@@ -34,7 +34,7 @@ jobs:
           # openapi-generator
           wget https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/5.3.0/openapi-generator-cli-5.3.0.jar -O openapi-generator-cli.jar
 
-      - name: "build image"
+      - name: 'build image'
         run: |
           docker build \
             --build-arg SM_ENVIRONMENT=local \

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -20,12 +20,12 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: '3.11'
 
       - uses: actions/setup-java@v4
         with:
-          distribution: "temurin" # See 'Supported distributions' for available options
-          java-version: "17"
+          distribution: 'temurin' # See 'Supported distributions' for available options
+          java-version: '17'
 
       - name: Setup build env
         run: |
@@ -46,21 +46,21 @@ jobs:
           unzip -o -d liquibase liquibase-${VERSION}.zip
           echo "$(pwd)/liquibase" >> $GITHUB_PATH
 
-      - name: "build image"
+      - name: 'build image'
         run: |
           docker build \
             --build-arg SM_ENVIRONMENT=local \
             --tag $SM_DOCKER \
             -f deploy/api/Dockerfile \
             .
 
-      - name: "build deployable API"
+      - name: 'build deployable API'
         run: |
           export OPENAPI_COMMAND="java -jar openapi-generator-cli.jar"
           python regenerate_api.py
           pip install .
 
-      - name: "Run unit tests"
+      - name: 'Run unit tests'
         id: runtests
         run: |
           coverage run -m pytest --doctest-modules --doctest-continue-on-failure test/
@@ -69,13 +69,13 @@ jobs:
 
           echo "rc=$rc" >> $GITHUB_OUTPUT
 
-      - name: "Upload coverage report"
+      - name: 'Upload coverage report'
         uses: codecov/codecov-action@v4
         with:
           files: ./coverage.xml
           token: ${{ secrets.CODECOV_TOKEN }}
 
-      - name: "build web front-end"
+      - name: 'build web front-end'
         run: |
           set -eo pipefail
           pushd web

diff --git a/.github/workflows/trigger_schema_updater.yaml b/.github/workflows/trigger_schema_updater.yaml
@@ -23,26 +23,26 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - id: "branch-check"
+      - id: 'branch-check'
         name: Fail if branch is not main or dev
-        if: github.event_name == 'workflow_dispatch' && (github.ref != 'refs/heads/main' && github.ref != 'refs/heads/dev')
+        if: github.event_name == 'workflow_dispatch' && (github.ref != 'refs/heads/main' && github.ref != 'refs/heads/dev' && github.ref != 'refs/heads/first-class-files-v3')
         run: |
           echo "This workflow should not be triggered with workflow_dispatch on a branch other than main or dev"
           exit 1
 
-      - id: "google-cloud-auth"
-        name: "Authenticate to Google Cloud"
-        uses: "google-github-actions/auth@v2"
+      - id: 'google-cloud-auth'
+        name: 'Authenticate to Google Cloud'
+        uses: 'google-github-actions/auth@v2'
         with:
-          workload_identity_provider: "projects/774248915715/locations/global/workloadIdentityPools/github-pool/providers/github-provider"
-          service_account: "[email protected]"
+          workload_identity_provider: 'projects/774248915715/locations/global/workloadIdentityPools/github-pool/providers/github-provider'
+          service_account: '[email protected]'
 
-      - id: "google-cloud-sdk-setup"
-        name: "Set up Cloud SDK"
+      - id: 'google-cloud-sdk-setup'
+        name: 'Set up Cloud SDK'
         uses: google-github-actions/setup-gcloud@v2
 
-      - id: "invoke-cloud-run"
-        name: "Invoke Cloud Run"
+      - id: 'invoke-cloud-run'
+        name: 'Invoke Cloud Run'
         run: |
           CLOUD_RUN_URL=$(gcloud run services describe schema-updater --region australia-southeast1 --format 'value(status.url)')/execute-liquibase?environment=${{ github.event.inputs.environment }}
           TOKEN=$(gcloud auth print-identity-token --impersonate-service-account="[email protected]" --audiences="$CLOUD_RUN_URL" --include-email)

diff --git a/api/graphql/schema.py b/api/graphql/schema.py
@@ -529,6 +529,7 @@ class GraphQLAnalysis:
     type: str
     status: strawberry.enum(AnalysisStatus)  # type: ignore
     output: str | None
+    outputs: strawberry.scalars.JSON | None
     timestamp_completed: datetime.datetime | None = None
     active: bool
     meta: strawberry.scalars.JSON | None
@@ -544,6 +545,7 @@ def from_internal(internal: AnalysisInternal) -> 'GraphQLAnalysis':
             type=internal.type,
             status=internal.status,
             output=internal.output,
+            outputs=internal.outputs,
             timestamp_completed=internal.timestamp_completed,
             active=internal.active,
             meta=internal.meta,

diff --git a/api/routes/analysis.py b/api/routes/analysis.py
@@ -44,6 +44,7 @@ class AnalysisModel(BaseModel):
     status: AnalysisStatus
     meta: dict[str, Any] | None = None
     output: str | None = None
+    outputs: dict[str, Any] | None = None
     active: bool = True
     # please don't use this, unless you're the analysis-runner,
     # the usage is tracked ... (Ծ_Ծ)
@@ -55,6 +56,7 @@ class AnalysisUpdateModel(BaseModel):
 
     status: AnalysisStatus
     output: str | None = None
+    outputs: dict[str, Any] | None = None
     meta: dict[str, Any] | None = None
     active: bool | None = None
 
@@ -69,6 +71,8 @@ class AnalysisQueryModel(BaseModel):
     status: AnalysisStatus | None = None
     meta: dict[str, Any] | None = None
     output: str | None = None
+    # Need to consider how we might filter outputs that are dicts
+    # outputs: str | dict | None = None
     active: bool | None = None
 
     def to_filter(self, project_id_map: dict[str, int]) -> AnalysisFilter:

diff --git a/api/server.py b/api/server.py
@@ -25,7 +25,7 @@
 from db.python.utils import get_logger
 
 # This tag is automatically updated by bump2version
-_VERSION = '7.3.3'
+_VERSION = '7.4.0'
 
 
 logger = get_logger()

diff --git a/api/settings.py b/api/settings.py
@@ -16,6 +16,7 @@
 PROFILE_REQUESTS_OUTPUT = os.getenv('SM_PROFILE_REQUESTS_OUTPUT', 'text').lower()
 IGNORE_GCP_CREDENTIALS_ERROR = os.getenv('SM_IGNORE_GCP_CREDENTIALS_ERROR') in TRUTH_SET
 MEMBERS_CACHE_LOCATION = os.getenv('SM_MEMBERS_CACHE_LOCATION')
+METAMIST_GCP_PROJECT = os.getenv('METAMIST_GCP_PROJECT')
 
 SEQR_URL = os.getenv('SM_SEQR_URL')
 SEQR_AUDIENCE = os.getenv('SM_SEQR_AUDIENCE')

diff --git a/db/project.xml b/db/project.xml
@@ -1509,9 +1509,90 @@
 		</sql>
 
 	</changeSet>
-
-
-
-
-
+	<changeSet id="2024-06-26_output_file" author="yash.pankhania">
+		<sql>SET @@system_versioning_alter_history = 1;</sql>
+		<createTable tableName="output_file">
+			<column name="id" type="INT" autoIncrement="true">
+				<constraints primaryKey="true" nullable="false" />
+			</column>
+			<column name="path" type="VARCHAR(255)">
+				<constraints
+					nullable="false"
+					unique="true"
+				/>
+			</column>
+			<column name="basename" type="VARCHAR(255)">
+				<constraints
+					nullable="false"
+				/>
+			</column>
+			<column name="dirname" type="VARCHAR(100)">
+				<constraints
+					nullable="false"
+				/>
+			</column>
+			<column name="nameroot" type="VARCHAR(255)">
+				<constraints
+					nullable="false"
+				/>
+			</column>
+			<column name="nameext" type="VARCHAR(25)">
+				<constraints
+					nullable="true"
+				/>
+			</column>
+			<column name="file_checksum" type="VARCHAR(255)">
+				<constraints
+					nullable="true"
+				/>
+			</column>
+			<column name="size" type="BIGINT">
+				<constraints
+					nullable="false"
+				/>
+			</column>
+			<column name="meta" type="VARCHAR(255)">
+				<constraints
+					nullable="true"
+				/>
+			</column>
+			<column name="valid" type="BOOLEAN" />
+			<column name="parent_id" type="INT">
+				<constraints
+					nullable="true"
+					foreignKeyName="FK_SECONDARY_FILE_PARENT_ID"
+					references="output_file(id)"
+				/>
+			</column>
+		</createTable>
+		<createTable tableName="analysis_outputs">
+			<column name="analysis_id" type="INT">
+				<constraints
+				nullable="false"
+				foreignKeyName="FK_ANALYSIS_OUTPUTS_ANALYSIS_ID"
+				references="analysis(id)"
+				/>
+			</column>
+			<column name="file_id" type="INT">
+				<constraints
+				nullable="true"
+				foreignKeyName="FK_ANALYSIS_OUTPUTS_FILE_ID"
+				references="output_file(id)"
+				/>
+			</column>
+			<column name="output" type="VARCHAR(255)">
+				<constraints
+				nullable="true"
+				/>
+			</column>
+			<column name="json_structure" type="VARCHAR(255)">
+				<constraints
+					nullable="true"
+				/>
+			</column>
+		</createTable>
+		<sql>ALTER TABLE `output_file` ADD SYSTEM VERSIONING;</sql>
+		<sql>ALTER TABLE `analysis_outputs` ADD SYSTEM VERSIONING;</sql>
+		<sql>ALTER TABLE `analysis_outputs` ADD CONSTRAINT `chk_file_id_output` CHECK ((file_id IS NOT NULL AND output IS NULL) OR (file_id IS NULL AND output IS NOT NULL));</sql>
+	</changeSet>
 </databaseChangeLog>
diff --git a/db/python/connect.py b/db/python/connect.py
@@ -50,6 +50,7 @@
     'cohort_sequencing_group',
     'analysis_cohort',
     'analysis_runner',
+    'analysis_outputs',
 ][::-1]