Release - family external ids and misc fixes

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 7.4.3
+current_version = 7.5.0
 commit = True
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>[A-z0-9-]+)

.github/workflows/deploy.yaml

@@ -12,8 +12,17 @@ permissions:
   contents: read
 
 jobs:
+  unittests:
+    uses: './.github/workflows/test.yaml'
+    secrets:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+      SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+      SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
+
   deploy:
+    name: Deploy
     runs-on: ubuntu-latest
+    needs: unittests
     environment: production
     env:
       DOCKER_BUILDKIT: 1

.github/workflows/test.yaml

@@ -1,11 +1,18 @@
 name: Test
-on: push
+on:
+  push:
+  workflow_call:
+    secrets:
+      CODECOV_TOKEN:
+        required: true
+      SONAR_TOKEN:
+        required: true
+      SONAR_HOST_URL:
+        required: true
 
 jobs:
   unittests:
     name: Run unit tests
-    # Run on merge to main, where the commit name starts with "Bump version:" (for bump2version)
-    #    if: "startsWith(github.event.head_commit.message, 'Bump version:')"
     runs-on: ubuntu-latest
     env:
       DOCKER_BUILDKIT: 1

.gitignore

@@ -68,3 +68,5 @@ profiles
 
 # sonarqube
 .scannerwork/
+
+latest_backup/

api/graphql/schema.py

@@ -728,6 +728,7 @@ class GraphQLFamily:
 
     id: int
     external_id: str
+    external_ids: strawberry.scalars.JSON
 
     description: str | None
     coded_phenotype: str | None
@@ -739,7 +740,8 @@ class GraphQLFamily:
     def from_internal(internal: FamilyInternal) -> 'GraphQLFamily':
         return GraphQLFamily(
             id=internal.id,
-            external_id=internal.external_id,
+            external_id=internal.external_ids[PRIMARY_EXTERNAL_ORG],
+            external_ids=internal.external_ids or {},
             description=internal.description,
             coded_phenotype=internal.coded_phenotype,
             project_id=internal.project,

api/routes/cohort.py

@@ -3,7 +3,12 @@
 from api.utils.db import Connection, get_project_db_connection
 from db.python.layers.cohort import CohortLayer
 from models.models.cohort import CohortBody, CohortCriteria, CohortTemplate, NewCohort
-from models.models.project import ProjectId, ProjectMemberRole, ReadAccessRoles
+from models.models.project import (
+    FullWriteAccessRoles,
+    ProjectId,
+    ProjectMemberRole,
+    ReadAccessRoles,
+)
 from models.utils.cohort_template_id_format import (
     cohort_template_id_format,
     cohort_template_id_transform_to_raw,
@@ -85,7 +90,7 @@ async def create_cohort_template(
     if template.criteria.projects:
         projects_for_criteria = connection.get_and_check_access_to_projects_for_names(
             project_names=template.criteria.projects,
-            allowed_roles=ReadAccessRoles,
+            allowed_roles=FullWriteAccessRoles,
         )
         criteria_project_ids = [p.id for p in projects_for_criteria if p.id]
 

api/routes/family.py

@@ -30,7 +30,7 @@ class FamilyUpdateModel(BaseModel):
     """Model for updating a family"""
 
     id: int
-    external_id: str | None = None
+    external_ids: dict[str, str] | None = None
     description: str | None = None
     coded_phenotype: str | None = None
 
@@ -171,7 +171,7 @@ async def update_family(
     return {
         'success': await family_layer.update_family(
             id_=family.id,
-            external_id=family.external_id,
+            external_ids=family.external_ids,
             description=family.description,
             coded_phenotype=family.coded_phenotype,
         )

api/server.py

@@ -25,7 +25,7 @@
 from db.python.utils import get_logger
 
 # This tag is automatically updated by bump2version
-_VERSION = '7.4.3'
+_VERSION = '7.5.0'
 
 
 logger = get_logger()

db/deploy/Dockerfile

@@ -8,17 +8,31 @@ ENV LIQUIBASE_VERSION=4.26.0
 ENV MARIADB_JDBC_VERSION=3.0.3
 
 # Install system dependencies
-RUN apt-get update && apt-get install -y --no-install-recommends gcc git ssh default-jdk wget unzip
+RUN apt-get update && apt-get install -y --no-install-recommends gcc git ssh default-jdk wget unzip \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create a non-root user and group
+RUN groupadd -r appuser && useradd -r -g appuser -d /home/appuser -m -s /bin/bash appuser
 
 # Download and install Liquibase
 RUN wget https://github.com/liquibase/liquibase/releases/download/v${LIQUIBASE_VERSION}/liquibase-${LIQUIBASE_VERSION}.zip \
     && unzip liquibase-${LIQUIBASE_VERSION}.zip -d /opt/liquibase \
     && chmod +x /opt/liquibase/liquibase \
-    # Clean up to reduce layer size
     && rm liquibase-${LIQUIBASE_VERSION}.zip \
+    # Clean up to reduce layer size
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
+# Change ownership of the liquibase directory to the non-root user
+RUN chown -R appuser:appuser /opt/liquibase
+
+# Switch to the non-root user
+USER appuser
+
+# Set the working directory
+WORKDIR /home/appuser
+
 # Download the MariaDB JDBC driver
 RUN wget https://downloads.mariadb.com/Connectors/java/connector-java-${MARIADB_JDBC_VERSION}/mariadb-java-client-${MARIADB_JDBC_VERSION}.jar
 RUN mv mariadb-java-client-${MARIADB_JDBC_VERSION}.jar /opt/

db/project.xml

@@ -1789,4 +1789,80 @@
 		<sql>ALTER TABLE `analysis_outputs` ADD SYSTEM VERSIONING;</sql>
 	</changeSet>
 
+	<changeSet id="2024-07-25-family-external-ids" author="john.marshall">
+		<createTable tableName="family_external_id">
+			<column name="project" type="INT">
+				<constraints
+					nullable="false"
+					foreignKeyName="FK_PROJECT_FAMILY_EXTERNAL_ID"
+					references="project(id)" />
+			</column>
+			<column name="family_id" type="INT">
+				<constraints
+					nullable="false"
+					foreignKeyName="FK_FAMILY_FAMILY_EXTERNAL_ID"
+					references="family(id)" />
+			</column>
+			<column name="name" type="VARCHAR(255)" />
+			<column name="external_id" type="VARCHAR(255)">
+				<constraints nullable="false" />
+			</column>
+			<column name="meta" type="LONGTEXT" />
+			<column name="audit_log_id" type="INT">
+				<constraints
+					nullable="false"
+					foreignKeyName="FK_FAMILY_EXTERNAL_ID_CHANGELOG_ID"
+					references="audit_log(id)" />
+			</column>
+		</createTable>
+		<addPrimaryKey
+			tableName="family_external_id"
+			columnNames="family_id,name"
+			constraintName="PK_FAMILY_EXTERNAL_ID"
+			validate="true"
+		/>
+		<addUniqueConstraint
+			tableName="family_external_id"
+			columnNames="project,external_id"
+			constraintName="UK_FAMILY_EXTERNAL_ID_UNIQUE_EIDS"
+			validate="true"
+		/>
+		<createIndex tableName="family" indexName="fk_project_family">
+			<column name="project" />
+		</createIndex>
+		<dropUniqueConstraint
+			tableName="family"
+			constraintName="UK_FAMILY_PROJECT_EXTERNALID"
+		/>
+
+		<sql>ALTER TABLE family_external_id ADD SYSTEM VERSIONING;</sql>
+
+		<!-- Migrate existing external_ids to the new tables, keyed by PRIMARY_EXTERNAL_ORG, i.e. '' -->
+		<sql>INSERT INTO audit_log (author, on_behalf_of, ar_guid, comment, auth_project, meta)
+		     VALUES ('liquibase', NULL, NULL, 'family external_id migration', NULL, NULL)
+		     RETURNING @audit_log_id := id;
+
+		     INSERT INTO family_external_id (project, family_id, name, external_id, audit_log_id)
+		       SELECT project, id, '', external_id, @audit_log_id
+		       FROM family;
+		</sql>
+	</changeSet>
+
+	<changeSet id="2024-07-26-drop-old-external-id-columns" author="john.marshall">
+		<sql>SET @@system_versioning_alter_history = 1;</sql>
+
+		<dropNotNullConstraint
+			tableName="family"
+			columnName="external_id"
+			columnDataType="VARCHAR(255)"
+		/>
+		<sql>UPDATE family SET external_id = NULL</sql>
+		<renameColumn
+			tableName="family"
+			oldColumnName="external_id"
+			newColumnName="_external_id_unused"
+			columnDataType="VARCHAR(255)"
+			remarks="Migration of family external IDs to separate table"
+		/>
+	</changeSet>
 </databaseChangeLog>

db/python/connect.py

@@ -23,7 +23,7 @@
 )
 from models.models.project import Project, ProjectId, ProjectMemberRole
 
-logging.basicConfig(level=logging.DEBUG)
+logging.basicConfig(level=logging.WARNING)
 logger = logging.getLogger(__name__)
 
 TABLES_ORDERED_BY_FK_DEPS = [
@@ -42,6 +42,7 @@
     'sample_external_id',
     'sequencing_group_external_id',
     'family',
+    'family_external_id',
     'family_participant',
     'participant_phenotypes',
     'group_member',

db/python/gcp_connect.py

@@ -10,7 +10,7 @@
 
 from db.python.utils import InternalError
 
-logging.basicConfig(level=logging.DEBUG)
+logging.basicConfig(level=logging.WARNING)
 logger = logging.getLogger(__name__)
 
 

db/python/layers/family.py

@@ -16,7 +16,7 @@
 from models.models import PRIMARY_EXTERNAL_ORG
 from models.models.family import FamilyInternal, PedRow, PedRowInternal
 from models.models.participant import ParticipantUpsertInternal
-from models.models.project import ProjectId, ReadAccessRoles
+from models.models.project import FullWriteAccessRoles, ProjectId, ReadAccessRoles
 
 
 class FamilyLayer(BaseLayer):
@@ -29,11 +29,14 @@ def __init__(self, connection: Connection):
         self.fptable = FamilyParticipantTable(self.connection)
 
     async def create_family(
-        self, external_id: str, description: str = None, coded_phenotype: str = None
+        self,
+        external_ids: dict[str, str],
+        description: str | None = None,
+        coded_phenotype: str | None = None,
     ):
         """Create a family"""
         return await self.ftable.create_family(
-            external_id=external_id,
+            external_ids=external_ids,
             description=description,
             coded_phenotype=coded_phenotype,
         )
@@ -127,20 +130,20 @@ async def get_families_by_participants(
     async def update_family(
         self,
         id_: int,
-        external_id: str = None,
+        external_ids: dict[str, str] | None = None,
         description: str = None,
         coded_phenotype: str = None,
     ) -> bool:
         """Update fields on some family"""
         project_ids = await self.ftable.get_projects_by_family_ids([id_])
 
         self.connection.check_access_to_projects_for_ids(
-            project_ids, allowed_roles=ReadAccessRoles
+            project_ids, allowed_roles=FullWriteAccessRoles
         )
 
         return await self.ftable.update_family(
             id_=id_,
-            external_id=external_id,
+            external_ids=external_ids,
             description=description,
             coded_phenotype=coded_phenotype,
         )
@@ -303,7 +306,7 @@ async def import_pedigree(
 
             for external_family_id in missing_external_family_ids:
                 internal_family_id = await self.ftable.create_family(
-                    external_id=external_family_id,
+                    external_ids={PRIMARY_EXTERNAL_ORG: external_family_id},
                     description=None,
                     coded_phenotype=None,
                 )

db/python/layers/participant.py

@@ -532,7 +532,7 @@ async def generic_individual_metadata_importer(
                 # they might not be missing
                 for external_family_id in missing_family_ids:
                     new_pid = await ftable.create_family(
-                        external_id=external_family_id,
+                        external_ids={PRIMARY_EXTERNAL_ORG: external_family_id},
                         description=None,
                         coded_phenotype=None,
                     )

db/python/layers/sample.py

@@ -287,7 +287,7 @@ async def upsert_samples(
         if sids:
             pjcts = await self.st.get_project_ids_for_sample_ids(sids)
             self.connection.check_access_to_projects_for_ids(
-                pjcts, allowed_roles=ReadAccessRoles
+                pjcts, allowed_roles=FullWriteAccessRoles
             )
 
         async with with_function():
@@ -438,7 +438,7 @@ async def get_history_of_sample(self, id_: int) -> list[SampleInternal]:
 
         projects = set(r.project for r in rows)
         self.connection.check_access_to_projects_for_ids(
-            projects, allowed_roles=FullWriteAccessRoles
+            projects, allowed_roles=ReadAccessRoles
         )
 
         return rows

db/python/layers/seqr.py

@@ -35,6 +35,7 @@
 from db.python.tables.project import Project
 from models.enums import AnalysisStatus
 from models.enums.web import SeqrDatasetType
+from models.models import PRIMARY_EXTERNAL_ORG
 
 # literally the most temporary thing ever, but for complete
 # automation need to have sample inclusion / exclusion
@@ -282,8 +283,8 @@ async def sync_families(
             return ['No families to synchronise']
         family_data = [
             {
-                'familyId': fam.external_id,
-                'displayName': fam.external_id,
+                'familyId': fam.external_ids[PRIMARY_EXTERNAL_ORG],
+                'displayName': fam.external_ids[PRIMARY_EXTERNAL_ORG],
                 'description': fam.description,
                 'codedPhenotype': fam.coded_phenotype,
             }

db/python/layers/web.py

@@ -343,7 +343,7 @@ def assemble_nested_participants_from(
             families = []
             for family in families_by_pid.get(participant.id, []):
                 families.append(
-                    FamilySimpleInternal(id=family.id, external_id=family.external_id)
+                    FamilySimpleInternal(id=family.id, external_ids=family.external_ids)
                 )
             nested_participant = NestedParticipantInternal(
                 id=participant.id,