OurDNA: Improve project table view and filtering (#742)

* Upgrade vite

* Add better filter options for participant

* Start to break apart project grids

* Stepping closer

* Move project summary to projectoverview

* More refactoring + filtering

* Bunch of small and large changes

* Quick and dirty project export button

* Export button + small updates

* Add startswith + fix tests + implement more + add :tired:

* Move ProjectGrid state into URL

* Minor fixes

* YAML support + code editor

* Move a bunch of stuff

* More code fixes + linting

* Fix created_on

* Improve ProjectGrid

* Previous commit I forgot?

* Linting

* Integrate external_ids more concretely

* Come back to failing test later

* Gotcha!

* Revert a previous change + linting

* Apply changes, fix the rest later

* Rename to filters

* Continue renaming filters

* Rewrite the projectparticipantgrid fields

* Linting

* Add re-orderable columns

* Apply review feedback

* Make the type machine happy

* Improve value filter rendering in participant grid (#834)

* tmp

* Linting

---------

Co-authored-by: Dan Coates <[email protected]>
Co-authored-by: Michael Franklin <[email protected]>

* Apply review feedback + standardise from_db_json

* Read the code before you make changes

* Update db/python/tables/participant.py

Co-authored-by: John Marshall <[email protected]>

---------

Co-authored-by: Michael Franklin <[email protected]>
Co-authored-by: Dan Coates <[email protected]>
Co-authored-by: John Marshall <[email protected]>

Author: 3 people

api/graphql/filters.py

@@ -2,7 +2,7 @@
 
 import strawberry
 
-from db.python.utils import GenericFilter, GenericMetaFilter
+from db.python.filters import GenericFilter, GenericMetaFilter
 
 T = TypeVar('T')
 Y = TypeVar('Y')
@@ -21,6 +21,7 @@ class GraphQLFilter(Generic[T]):
     lte: T | None = None
     contains: T | None = None
     icontains: T | None = None
+    startswith: T | None = None
 
     def all_values(self):
         """
@@ -45,6 +46,8 @@ def all_values(self):
             v.append(self.contains)
         if self.icontains:
             v.append(self.icontains)
+        if self.startswith:
+            v.append(self.startswith)
 
         return v
 
@@ -60,6 +63,7 @@ def to_internal_filter(self) -> GenericFilter[T]:
             lte=self.lte,
             contains=self.contains,
             icontains=self.icontains,
+            startswith=self.startswith,
         )
 
     def to_internal_filter_mapped(self, f: Callable[[T], Y]) -> GenericFilter[Y]:
@@ -77,6 +81,7 @@ def to_internal_filter_mapped(self, f: Callable[[T], Y]) -> GenericFilter[Y]:
             lte=f(self.lte) if self.lte else None,
             contains=f(self.contains) if self.contains else None,
             icontains=f(self.icontains) if self.icontains else None,
+            startswith=f(self.startswith) if self.startswith else None,
         )
 
 

api/graphql/loaders.py

@@ -10,6 +10,7 @@
 from strawberry.dataloader import DataLoader
 
 from api.utils import get_projectless_db_connection, group_by
+from db.python.filters import GenericFilter, get_hashable_value
 from db.python.layers import (
     AnalysisLayer,
     AssayLayer,
@@ -22,10 +23,11 @@
 from db.python.tables.analysis import AnalysisFilter
 from db.python.tables.assay import AssayFilter
 from db.python.tables.family import FamilyFilter
+from db.python.tables.participant import ParticipantFilter
 from db.python.tables.project import ProjectPermissionsTable
 from db.python.tables.sample import SampleFilter
 from db.python.tables.sequencing_group import SequencingGroupFilter
-from db.python.utils import GenericFilter, NotFoundError, get_hashable_value
+from db.python.utils import NotFoundError
 from models.models import (
     AnalysisInternal,
     AssayInternal,
@@ -254,7 +256,12 @@ async def load_sequencing_groups_for_samples(
     """
     sglayer = SequencingGroupLayer(connection)
     _filter = dataclasses.replace(filter) if filter else SequencingGroupFilter()
-    _filter.sample_id = GenericFilter(in_=ids)
+    if not _filter.sample:
+        _filter.sample = SequencingGroupFilter.SequencingGroupSampleFilter(
+            id=GenericFilter(in_=ids)
+        )
+    else:
+        _filter.sample.id = GenericFilter(in_=ids)
 
     sequencing_groups = await sglayer.query(_filter)
     sg_map = group_by(sequencing_groups, lambda sg: sg.sample_id)
@@ -380,22 +387,22 @@ async def load_participants_for_families(
     return [pmap.get(fid, []) for fid in family_ids]
 
 
-@connected_data_loader(LoaderKeys.PARTICIPANTS_FOR_PROJECTS)
+@connected_data_loader_with_params(
+    LoaderKeys.PARTICIPANTS_FOR_PROJECTS, default_factory=list
+)
 async def load_participants_for_projects(
-    project_ids: list[ProjectId], connection
-) -> list[list[ParticipantInternal]]:
+    ids: list[ProjectId], filter_: ParticipantFilter, connection
+) -> dict[ProjectId, list[ParticipantInternal]]:
     """
     Get all participants in a project
     """
 
-    retval: list[list[ParticipantInternal]] = []
-
-    for project in project_ids:
-        retval.append(
-            await ParticipantLayer(connection).get_participants(project=project)
-        )
+    f = copy.copy(filter_)
+    f.project = GenericFilter(in_=ids)
+    participants = await ParticipantLayer(connection).query(f)
 
-    return retval
+    pmap = group_by(participants, lambda p: p.project)
+    return pmap
 
 
 @connected_data_loader_with_params(

api/graphql/schema.py

@@ -20,6 +20,7 @@
 )
 from api.graphql.loaders import LoaderKeys, get_context
 from db.python import enum_tables
+from db.python.filters import GenericFilter
 from db.python.layers import (
     AnalysisLayer,
     AnalysisRunnerLayer,
@@ -35,10 +36,10 @@
 from db.python.tables.assay import AssayFilter
 from db.python.tables.cohort import CohortFilter, CohortTemplateFilter
 from db.python.tables.family import FamilyFilter
+from db.python.tables.participant import ParticipantFilter
 from db.python.tables.project import ProjectPermissionsTable
 from db.python.tables.sample import SampleFilter
 from db.python.tables.sequencing_group import SequencingGroupFilter
-from db.python.utils import GenericFilter
 from models.enums import AnalysisStatus
 from models.models import (
     PRIMARY_EXTERNAL_ORG,
@@ -331,10 +332,39 @@ async def families(
 
     @strawberry.field()
     async def participants(
-        self, info: Info, root: 'Project'
+        self,
+        info: Info,
+        root: 'GraphQLProject',
+        id: GraphQLFilter[int] | None = None,
+        external_id: GraphQLFilter[str] | None = None,
+        meta: GraphQLMetaFilter | None = None,
+        reported_sex: GraphQLFilter[int] | None = None,
+        reported_gender: GraphQLFilter[str] | None = None,
+        karyotype: GraphQLFilter[str] | None = None,
     ) -> list['GraphQLParticipant']:
         loader = info.context[LoaderKeys.PARTICIPANTS_FOR_PROJECTS]
-        participants = await loader.load(root.id)
+        participants = await loader.load(
+            {
+                'id': root.id,
+                'filter_': ParticipantFilter(
+                    project=GenericFilter(eq=root.id),
+                    id=id.to_internal_filter() if id else None,
+                    external_id=(
+                        external_id.to_internal_filter() if external_id else None
+                    ),
+                    meta=graphql_meta_filter_to_internal_filter(meta),
+                    reported_gender=(
+                        reported_gender.to_internal_filter()
+                        if reported_gender
+                        else None
+                    ),
+                    reported_sex=(
+                        reported_sex.to_internal_filter() if reported_sex else None
+                    ),
+                    karyotype=karyotype.to_internal_filter() if karyotype else None,
+                ),
+            }
+        )
         return [GraphQLParticipant.from_internal(p) for p in participants]
 
     @strawberry.field()
@@ -1181,8 +1211,10 @@ async def sequencing_groups(
 
         filter_ = SequencingGroupFilter(
             project=_project_filter,
-            sample_id=(
-                sample_id.to_internal_filter_mapped(sample_id_transform_to_raw)
+            sample=(
+                SequencingGroupFilter.SequencingGroupSampleFilter(
+                    id=sample_id.to_internal_filter_mapped(sample_id_transform_to_raw)
+                )
                 if sample_id
                 else None
             ),
@@ -1200,7 +1232,11 @@ async def sequencing_groups(
                 else GenericFilter(eq=True)
             ),
             created_on=created_on.to_internal_filter() if created_on else None,
-            assay_meta=assay_meta,
+            assay=(
+                SequencingGroupFilter.SequencingGroupAssayFilter(
+                    meta=graphql_meta_filter_to_internal_filter(assay_meta),
+                )
+            ),
             has_cram=has_cram,
             has_gvcf=has_gvcf,
         )

api/routes/analysis.py

@@ -16,10 +16,10 @@
     get_projectless_db_connection,
 )
 from api.utils.export import ExportType
+from db.python.filters import GenericFilter
 from db.python.layers.analysis import AnalysisLayer
 from db.python.tables.analysis import AnalysisFilter
 from db.python.tables.project import ProjectPermissionsTable
-from db.python.utils import GenericFilter
 from models.enums import AnalysisStatus
 from models.models.analysis import (
     Analysis,

api/routes/analysis_runner.py

@@ -7,9 +7,9 @@
     get_project_readonly_connection,
     get_project_write_connection,
 )
+from db.python.filters import GenericFilter
 from db.python.layers.analysis_runner import AnalysisRunnerLayer
 from db.python.tables.analysis_runner import AnalysisRunnerFilter
-from db.python.utils import GenericFilter
 from models.models.analysis_runner import AnalysisRunner, AnalysisRunnerInternal
 
 router = APIRouter(prefix='/analysis-runner', tags=['analysis-runner'])

api/routes/assay.py

@@ -4,10 +4,10 @@
 
 from api.utils import get_project_readonly_connection
 from api.utils.db import Connection, get_projectless_db_connection
+from db.python.filters import GenericFilter
 from db.python.layers.assay import AssayLayer
 from db.python.tables.assay import AssayFilter
 from db.python.tables.project import ProjectPermissionsTable
-from db.python.utils import GenericFilter
 from models.base import SMBase
 from models.models.assay import AssayUpsert
 from models.utils.sample_id_format import sample_id_transform_to_raw_list

api/routes/family.py

@@ -17,9 +17,9 @@
 )
 from api.utils.export import ExportType
 from api.utils.extensions import guess_delimiter_by_upload_file_obj
+from db.python.filters import GenericFilter
 from db.python.layers.family import FamilyLayer, PedRow
 from db.python.tables.family import FamilyFilter
-from db.python.utils import GenericFilter
 from models.models.family import Family
 from models.utils.sample_id_format import sample_id_transform_to_raw_list