diff --git a/api/graphql/schema.py b/api/graphql/schema.py index b424d8575..f849e3932 100644 --- a/api/graphql/schema.py +++ b/api/graphql/schema.py @@ -728,6 +728,7 @@ class GraphQLFamily: id: int external_id: str + external_ids: strawberry.scalars.JSON description: str | None coded_phenotype: str | None @@ -739,7 +740,8 @@ class GraphQLFamily: def from_internal(internal: FamilyInternal) -> 'GraphQLFamily': return GraphQLFamily( id=internal.id, - external_id=internal.external_id, + external_id=internal.external_ids[PRIMARY_EXTERNAL_ORG], + external_ids=internal.external_ids or {}, description=internal.description, coded_phenotype=internal.coded_phenotype, project_id=internal.project, diff --git a/api/routes/family.py b/api/routes/family.py index 6a5b966f1..c20cce3ce 100644 --- a/api/routes/family.py +++ b/api/routes/family.py @@ -30,7 +30,7 @@ class FamilyUpdateModel(BaseModel): """Model for updating a family""" id: int - external_id: str | None = None + external_ids: dict[str, str] | None = None description: str | None = None coded_phenotype: str | None = None @@ -171,7 +171,7 @@ async def update_family( return { 'success': await family_layer.update_family( id_=family.id, - external_id=family.external_id, + external_ids=family.external_ids, description=family.description, coded_phenotype=family.coded_phenotype, ) diff --git a/db/project.xml b/db/project.xml index a28dbb947..4b4e9a875 100644 --- a/db/project.xml +++ b/db/project.xml @@ -1789,4 +1789,80 @@ ALTER TABLE `analysis_outputs` ADD SYSTEM VERSIONING; + + + + + + + + + + + + + + + + + + + + + + + + + ALTER TABLE family_external_id ADD SYSTEM VERSIONING; + + + INSERT INTO audit_log (author, on_behalf_of, ar_guid, comment, auth_project, meta) + VALUES ('liquibase', NULL, NULL, 'family external_id migration', NULL, NULL) + RETURNING @audit_log_id := id; + + INSERT INTO family_external_id (project, family_id, name, external_id, audit_log_id) + SELECT project, id, '', external_id, @audit_log_id + FROM family; + + + + + SET @@system_versioning_alter_history = 1; + + + UPDATE family SET external_id = NULL + + diff --git a/db/python/connect.py b/db/python/connect.py index fd9e97e08..05a1dd5b7 100644 --- a/db/python/connect.py +++ b/db/python/connect.py @@ -42,6 +42,7 @@ 'sample_external_id', 'sequencing_group_external_id', 'family', + 'family_external_id', 'family_participant', 'participant_phenotypes', 'group_member', diff --git a/db/python/layers/family.py b/db/python/layers/family.py index 130915243..1d296ff4f 100644 --- a/db/python/layers/family.py +++ b/db/python/layers/family.py @@ -29,11 +29,14 @@ def __init__(self, connection: Connection): self.fptable = FamilyParticipantTable(self.connection) async def create_family( - self, external_id: str, description: str = None, coded_phenotype: str = None + self, + external_ids: dict[str, str], + description: str | None = None, + coded_phenotype: str | None = None, ): """Create a family""" return await self.ftable.create_family( - external_id=external_id, + external_ids=external_ids, description=description, coded_phenotype=coded_phenotype, ) @@ -127,7 +130,7 @@ async def get_families_by_participants( async def update_family( self, id_: int, - external_id: str = None, + external_ids: dict[str, str] | None = None, description: str = None, coded_phenotype: str = None, ) -> bool: @@ -140,7 +143,7 @@ async def update_family( return await self.ftable.update_family( id_=id_, - external_id=external_id, + external_ids=external_ids, description=description, coded_phenotype=coded_phenotype, ) @@ -303,7 +306,7 @@ async def import_pedigree( for external_family_id in missing_external_family_ids: internal_family_id = await self.ftable.create_family( - external_id=external_family_id, + external_ids={PRIMARY_EXTERNAL_ORG: external_family_id}, description=None, coded_phenotype=None, ) diff --git a/db/python/layers/participant.py b/db/python/layers/participant.py index 6456cd9a7..f29171ab9 100644 --- a/db/python/layers/participant.py +++ b/db/python/layers/participant.py @@ -532,7 +532,7 @@ async def generic_individual_metadata_importer( # they might not be missing for external_family_id in missing_family_ids: new_pid = await ftable.create_family( - external_id=external_family_id, + external_ids={PRIMARY_EXTERNAL_ORG: external_family_id}, description=None, coded_phenotype=None, ) diff --git a/db/python/layers/seqr.py b/db/python/layers/seqr.py index ae2c0cd7b..242fd5f51 100644 --- a/db/python/layers/seqr.py +++ b/db/python/layers/seqr.py @@ -35,6 +35,7 @@ from db.python.tables.project import Project from models.enums import AnalysisStatus from models.enums.web import SeqrDatasetType +from models.models import PRIMARY_EXTERNAL_ORG # literally the most temporary thing ever, but for complete # automation need to have sample inclusion / exclusion @@ -282,8 +283,8 @@ async def sync_families( return ['No families to synchronise'] family_data = [ { - 'familyId': fam.external_id, - 'displayName': fam.external_id, + 'familyId': fam.external_ids[PRIMARY_EXTERNAL_ORG], + 'displayName': fam.external_ids[PRIMARY_EXTERNAL_ORG], 'description': fam.description, 'codedPhenotype': fam.coded_phenotype, } diff --git a/db/python/layers/web.py b/db/python/layers/web.py index e217b37a0..beba36330 100644 --- a/db/python/layers/web.py +++ b/db/python/layers/web.py @@ -343,7 +343,7 @@ def assemble_nested_participants_from( families = [] for family in families_by_pid.get(participant.id, []): families.append( - FamilySimpleInternal(id=family.id, external_id=family.external_id) + FamilySimpleInternal(id=family.id, external_ids=family.external_ids) ) nested_participant = NestedParticipantInternal( id=participant.id, diff --git a/db/python/tables/family.py b/db/python/tables/family.py index 6e77a249f..a8dab8fee 100644 --- a/db/python/tables/family.py +++ b/db/python/tables/family.py @@ -5,8 +5,7 @@ from db.python.filters import GenericFilter, GenericFilterModel from db.python.tables.base import DbBase from db.python.utils import NotFoundError, escape_like_term -from models.models.family import FamilyInternal -from models.models.project import ProjectId +from models.models import PRIMARY_EXTERNAL_ORG, FamilyInternal, ProjectId @dataclasses.dataclass @@ -54,15 +53,21 @@ async def query( ) -> tuple[set[ProjectId], list[FamilyInternal]]: """Get all families for some project""" _query = """ - SELECT f.id, f.external_id, f.description, f.coded_phenotype, f.project + SELECT f.id, JSON_OBJECTAGG(feid.name, feid.external_id) AS external_ids, + f.description, f.coded_phenotype, f.project FROM family f + INNER JOIN family_external_id feid ON f.id = feid.family_id """ if not filter_.project and not filter_.id: raise ValueError('Project or ID filter is required for family queries') has_participant_join = False - field_overrides = {'id': 'f.id', 'external_id': 'f.external_id'} + field_overrides = { + 'id': 'f.id', + 'external_id': 'feid.external_id', + 'project': 'f.project', + } has_participant_join = False if filter_.participant_id: @@ -87,6 +92,10 @@ async def query( if wheres: _query += f'WHERE {wheres}' + _query += """ + GROUP BY f.id, f.description, f.coded_phenotype, f.project + """ + rows = await self.connection.fetch_all(_query, values) seen = set() families = [] @@ -107,10 +116,13 @@ async def get_families_by_participants( return set(), {} _query = """ - SELECT id, external_id, description, coded_phenotype, project, fp.participant_id - FROM family - INNER JOIN family_participant fp ON family.id = fp.family_id + SELECT f.id, JSON_OBJECTAGG(feid.name, feid.external_id) AS external_ids, + f.description, f.coded_phenotype, f.project, fp.participant_id + FROM family f + INNER JOIN family_external_id feid ON f.id = feid.family_id + INNER JOIN family_participant fp ON f.id = fp.family_id WHERE fp.participant_id in :pids + GROUP BY f.id, f.description, f.coded_phenotype, f.project, fp.participant_id """ ret_map = defaultdict(list) projects: set[ProjectId] = set() @@ -129,8 +141,8 @@ async def search( Search by some term, return [ProjectId, FamilyId, ExternalId] """ _query = """ - SELECT project, id, external_id - FROM family + SELECT project, family_id, external_id + FROM family_external_id WHERE project in :project_ids AND external_id LIKE :search_pattern LIMIT :limit """ @@ -142,7 +154,7 @@ async def search( 'limit': limit, }, ) - return [(r['project'], r['id'], r['external_id']) for r in rows] + return [(r['project'], r['family_id'], r['external_id']) for r in rows] async def get_family_external_ids_by_participant_ids( self, participant_ids @@ -152,9 +164,9 @@ async def get_family_external_ids_by_participant_ids( return {} _query = """ - SELECT f.external_id, fp.participant_id + SELECT feid.external_id, fp.participant_id FROM family_participant fp - INNER JOIN family f ON fp.family_id = f.id + INNER JOIN family_external_id feid ON fp.family_id = feid.family_id WHERE fp.participant_id in :pids """ rows = await self.connection.fetch_all(_query, {'pids': participant_ids}) @@ -167,31 +179,78 @@ async def get_family_external_ids_by_participant_ids( async def update_family( self, id_: int, - external_id: str | None = None, + external_ids: dict[str, str | None] | None = None, description: str | None = None, coded_phenotype: str | None = None, ) -> bool: """Update values for a family""" - values: Dict[str, Any] = {'audit_log_id': await self.audit_log_id()} - if external_id: - values['external_id'] = external_id + audit_log_id = await self.audit_log_id() + + values: Dict[str, Any] = {'audit_log_id': audit_log_id} if description: values['description'] = description if coded_phenotype: values['coded_phenotype'] = coded_phenotype - setters = ', '.join(f'{field} = :{field}' for field in values) - _query = f""" -UPDATE family -SET {setters} -WHERE id = :id - """ - await self.connection.execute(_query, {**values, 'id': id_}) + async with self.connection.transaction(): + if external_ids is None: + external_ids = {} + + to_delete = [k.lower() for k, v in external_ids.items() if v is None] + to_update = {k.lower(): v for k, v in external_ids.items() if v is not None} + + if to_delete: + await self.connection.execute( + """ + -- Set audit_log_id to this transaction before deleting the rows + UPDATE family_external_id + SET audit_log_id = :audit_log_id + WHERE family_id = :id AND name IN :names; + + DELETE FROM family_external_id + WHERE family_id = :id AND name in :names + """, + {'id': id_, 'names': to_delete, 'audit_log_id': audit_log_id}, + ) + + if to_update: + project = await self.connection.fetch_val( + 'SELECT project FROM family WHERE id = :id', + {'id': id_}, + ) + + _update_query = """ + INSERT INTO family_external_id (project, family_id, name, external_id, audit_log_id) + VALUES (:project, :id, :name, :external_id, :audit_log_id) + ON DUPLICATE KEY UPDATE external_id = :external_id, audit_log_id = :audit_log_id + """ + _update_values = [ + { + 'project': project, + 'id': id_, + 'name': name, + 'external_id': eid, + 'audit_log_id': audit_log_id, + } + for name, eid in to_update.items() + ] + await self.connection.execute_many(_update_query, _update_values) + + setters = ', '.join(f'{field} = :{field}' for field in values) + await self.connection.execute( + f""" + UPDATE family + SET {setters} + WHERE id = :id + """, + {**values, 'id': id_}, + ) + return True async def create_family( self, - external_id: str, + external_ids: dict[str, str], description: Optional[str], coded_phenotype: Optional[str], project: ProjectId | None = None, @@ -199,25 +258,41 @@ async def create_family( """ Create a new sample, and add it to database """ - updater = { - 'external_id': external_id, - 'description': description, - 'coded_phenotype': coded_phenotype, - 'audit_log_id': await self.audit_log_id(), - 'project': project or self.project_id, - } - keys = list(updater.keys()) - str_keys = ', '.join(keys) - placeholder_keys = ', '.join(f':{k}' for k in keys) - _query = f""" -INSERT INTO family - ({str_keys}) -VALUES - ({placeholder_keys}) -RETURNING id - """ + audit_log_id = await self.audit_log_id() + + async with self.connection.transaction(): + new_id = await self.connection.fetch_val( + """ + INSERT INTO family (project, description, coded_phenotype, audit_log_id) + VALUES (:project, :description, :coded_phenotype, :audit_log_id) + RETURNING id + """, + { + 'project': project or self.project_id, + 'description': description, + 'coded_phenotype': coded_phenotype, + 'audit_log_id': audit_log_id, + }, + ) + + await self.connection.execute_many( + """ + INSERT INTO family_external_id (project, family_id, name, external_id, audit_log_id) + VALUES (:project, :family_id, :name, :external_id, :audit_log_id) + """, + [ + { + 'project': project or self.project_id, + 'family_id': new_id, + 'name': name, + 'external_id': eid, + 'audit_log_id': audit_log_id, + } + for name, eid in external_ids.items() + ], + ) - return await self.connection.fetch_val(_query, updater) + return new_id async def insert_or_update_multiple_families( self, @@ -226,35 +301,65 @@ async def insert_or_update_multiple_families( coded_phenotypes: List[Optional[str]], project: ProjectId | None = None, ): - """Upsert""" - updater = [ - { - 'external_id': eid, - 'description': descr, - 'coded_phenotype': cph, - 'audit_log_id': await self.audit_log_id(), - 'project': project or self.project_id, - } - for eid, descr, cph in zip(external_ids, descriptions, coded_phenotypes) - ] - - keys = list(updater[0].keys()) - str_keys = ', '.join(keys) - placeholder_keys = ', '.join(f':{k}' for k in keys) - - update_only_keys = [k for k in keys if k not in ('external_id', 'project')] - str_uo_placeholder_keys = ', '.join(f'{k} = :{k}' for k in update_only_keys) - - _query = f"""\ -INSERT INTO family - ({str_keys}) -VALUES - ({placeholder_keys}) -ON DUPLICATE KEY UPDATE - {str_uo_placeholder_keys} """ + Upsert several families. + At present, this function only supports upserting the primary external id. + """ + audit_log_id = await self.audit_log_id() + + for eid, descr, cph in zip(external_ids, descriptions, coded_phenotypes): + existing_id = await self.connection.fetch_val( + """ + SELECT family_id FROM family_external_id + WHERE project = :project AND external_id = :external_id + """, + {'project': project or self.project_id, 'external_id': eid}, + ) + + if existing_id is None: + new_id = await self.connection.fetch_val( + """ + INSERT INTO family (project, description, coded_phenotype, audit_log_id) + VALUES (:project, :description, :coded_phenotype, :audit_log_id) + RETURNING id + """, + { + 'project': project or self.project_id, + 'description': descr, + 'coded_phenotype': cph, + 'audit_log_id': audit_log_id, + }, + ) + await self.connection.execute( + """ + INSERT INTO family_external_id (project, family_id, name, external_id, audit_log_id) + VALUES (:project, :family_id, :name, :external_id, :audit_log_id) + """, + { + 'project': project or self.project_id, + 'family_id': new_id, + 'name': PRIMARY_EXTERNAL_ORG, + 'external_id': eid, + 'audit_log_id': audit_log_id, + }, + ) + + else: + await self.connection.execute( + """ + UPDATE family + SET description = :description, coded_phenotype = :coded_phenotype, + audit_log_id = :audit_log_id + WHERE id = :id + """, + { + 'id': existing_id, + 'description': descr, + 'coded_phenotype': cph, + 'audit_log_id': audit_log_id, + }, + ) - await self.connection.execute_many(_query, updater) return True async def get_id_map_by_external_ids( @@ -265,9 +370,12 @@ async def get_id_map_by_external_ids( if not family_ids: return {} - _query = 'SELECT external_id, id FROM family WHERE external_id in :external_ids AND project = :project' results = await self.connection.fetch_all( - _query, {'external_ids': family_ids, 'project': project or self.project_id} + """ + SELECT external_id, family_id AS id FROM family_external_id + WHERE external_id in :external_ids AND project = :project + """, + {'external_ids': family_ids, 'project': project or self.project_id}, ) id_map = {r['external_id']: r['id'] for r in results} @@ -288,19 +396,26 @@ async def get_id_map_by_external_ids( async def get_id_map_by_internal_ids( self, family_ids: List[int], allow_missing=False ): - """Get map of {external_id: internal_id} for a family""" + """Get map of {internal_id: primary_external_id} for a family""" if len(family_ids) == 0: return {} - _query = 'SELECT id, external_id FROM family WHERE id in :ids' - results = await self.connection.fetch_all(_query, {'ids': family_ids}) - id_map = {r['id']: r['external_id'] for r in results} + + results = await self.connection.fetch_all( + """ + SELECT family_id, external_id + FROM family_external_id + WHERE family_id in :ids AND name = :PRIMARY_EXTERNAL_ORG + """, + {'ids': family_ids, 'PRIMARY_EXTERNAL_ORG': PRIMARY_EXTERNAL_ORG}, + ) + id_map = {r['family_id']: r['external_id'] for r in results} if not allow_missing and len(id_map) != len(family_ids): - provided_external_ids = set(family_ids) + provided_internal_ids = set(family_ids) # do the check again, but use the set this time # (in case we're provided a list with duplicates) - if len(id_map) != len(provided_external_ids): + if len(id_map) != len(provided_internal_ids): # we have families missing from the map, so we'll 404 the whole thing - missing_family_ids = provided_external_ids - set(id_map.keys()) + missing_family_ids = provided_internal_ids - set(id_map.keys()) raise NotFoundError( f"Couldn't find families with internal IDS: {', '.join(str(m) for m in missing_family_ids)}" diff --git a/db/python/tables/project.py b/db/python/tables/project.py index 6efd859d9..cd064833d 100644 --- a/db/python/tables/project.py +++ b/db/python/tables/project.py @@ -242,6 +242,7 @@ async def delete_project_data(self, project_id: int, delete_project: bool) -> bo DELETE FROM family_participant WHERE family_id IN ( SELECT id FROM family where project = :project ); +DELETE FROM family_external_id WHERE project = :project; DELETE FROM family WHERE project = :project; DELETE FROM sequencing_group_external_id WHERE project = :project; DELETE FROM sample_external_id WHERE project = :project; diff --git a/models/models/family.py b/models/models/family.py index 6113f27d8..b55d812d9 100644 --- a/models/models/family.py +++ b/models/models/family.py @@ -1,27 +1,27 @@ import logging -from pydantic import BaseModel +from models.base import SMBase, parse_sql_dict -class FamilySimpleInternal(BaseModel): +class FamilySimpleInternal(SMBase): """Simple family model for internal use""" id: int - external_id: str + external_ids: dict[str, str] def to_external(self): """Convert to external model""" return FamilySimple( id=self.id, - external_id=self.external_id, + external_ids=self.external_ids, ) -class FamilyInternal(BaseModel): +class FamilyInternal(SMBase): """Family model""" id: int - external_id: str + external_ids: dict[str, str] project: int description: str | None = None coded_phenotype: str | None = None @@ -29,31 +29,32 @@ class FamilyInternal(BaseModel): @staticmethod def from_db(d): """From DB fields""" - return FamilyInternal(**d) + external_ids = parse_sql_dict(d.pop('external_ids', {})) + return FamilyInternal(**d, external_ids=external_ids) def to_external(self): """Convert to external model""" return Family( id=self.id, - external_id=self.external_id, + external_ids=self.external_ids, project=self.project, description=self.description, coded_phenotype=self.coded_phenotype, ) -class FamilySimple(BaseModel): +class FamilySimple(SMBase): """Simple family model, mostly for web access""" id: int - external_id: str + external_ids: dict[str, str] -class Family(BaseModel): +class Family(SMBase): """Family model""" id: int | None - external_id: str + external_ids: dict[str, str] project: int description: str | None = None coded_phenotype: str | None = None @@ -62,7 +63,7 @@ def to_internal(self): """Convert to internal model""" return FamilyInternal( id=self.id, - external_id=self.external_id, + external_ids=self.external_ids, project=self.project, description=self.description, coded_phenotype=self.coded_phenotype, diff --git a/test/test_comment.py b/test/test_comment.py index ce0530940..4c350f9b2 100644 --- a/test/test_comment.py +++ b/test/test_comment.py @@ -626,7 +626,9 @@ async def test_add_comment_to_participant(self): async def test_add_comment_to_family(self): """Test adding a comment to an family""" - family = await self.flayer.create_family(external_id='f_external_id') + family = await self.flayer.create_family( + external_ids={PRIMARY_EXTERNAL_ORG: 'f_external_id'}, + ) comment_text = 'Family Test Comment 1234' created_comment = await self.add_comment_to_family(family, comment_text) @@ -823,7 +825,9 @@ async def test_deleting_and_restoring_comment(self): async def test_sample_discussion_related_comments(self): """Test getting related comments""" - family_id = await self.flayer.create_family(external_id='f_external_id') + family_id = await self.flayer.create_family( + external_ids={PRIMARY_EXTERNAL_ORG: 'f_external_id'}, + ) # This will create participant, sample, assay, sequencing group participant = await self.player.upsert_participant(get_participant_to_insert()) diff --git a/test/test_external_id.py b/test/test_external_id.py index c62468ba2..b248c393f 100644 --- a/test/test_external_id.py +++ b/test/test_external_id.py @@ -2,7 +2,9 @@ from pymysql.err import IntegrityError +from db.python.filters import GenericFilter from db.python.layers import FamilyLayer, ParticipantLayer, SampleLayer +from db.python.tables.family import FamilyFilter, FamilyTable from db.python.utils import NotFoundError from models.models import ( PRIMARY_EXTERNAL_ORG, @@ -170,7 +172,7 @@ async def test_fill_in_missing(self): async def test_get_by_families(self): """Exercise get_participants_by_families() method""" flayer = FamilyLayer(self.connection) - fid = await flayer.create_family(external_id='Jones') + fid = await flayer.create_family(external_ids={'org': 'Jones'}) child = await self.player.upsert_participant( ParticipantUpsertInternal( @@ -193,6 +195,35 @@ async def test_get_by_families(self): result[fid][0].external_ids, {PRIMARY_EXTERNAL_ORG: 'P20', 'd': 'D20'} ) + @run_as_sync + async def test_get_families_by_participants(self): + """Exercise FamilyLayer's get_families_by_participants() method""" + flayer = FamilyLayer(self.connection) + fid = await flayer.create_family( + external_ids={PRIMARY_EXTERNAL_ORG: 'Smith'}, + description='Blacksmiths', + coded_phenotype='burnt', + ) + + child = await self.player.upsert_participant( + ParticipantUpsertInternal( + external_ids={PRIMARY_EXTERNAL_ORG: 'P20', 'd': 'D20'} + ), + ) + + await self.player.add_participant_to_family( + family_id=fid, + participant_id=child.id, + paternal_id=self.p1.id, + maternal_id=self.p2.id, + affected=2, + ) + + result = await flayer.get_families_by_participants([child.id, self.p1.id]) + self.assertEqual(len(result), 1) + self.assertEqual(len(result[child.id]), 1) + self.assertEqual(result[child.id][0].description, 'Blacksmiths') + @run_as_sync async def test_update_many(self): """Exercise update_many_participant_external_ids() method""" @@ -423,3 +454,120 @@ async def test_get_history(self): self.assertDictEqual(result[1].meta, {'foo': 'bar'}) self.assertDictEqual(result[2].meta, {'foo': 'bar', 'fruit': 'banana'}) self.assertDictEqual(result[2].meta, sample.meta) + + +class TestFamily(DbIsolatedTest): + """Test family external ids""" + + @run_as_sync + async def setUp(self): + super().setUp() + self.flayer = FamilyLayer(self.connection) + + @run_as_sync + async def test_create_update(self): + """Exercise create_family() and update_family() methods""" + family_id = await self.flayer.create_family( + external_ids={PRIMARY_EXTERNAL_ORG: 'Smith'}, + description='Blacksmiths', + coded_phenotype='burnt', + ) + + family = await self.flayer.get_family_by_internal_id(family_id) + self.assertDictEqual(family.external_ids, {PRIMARY_EXTERNAL_ORG: 'Smith'}) + self.assertEqual(family.description, 'Blacksmiths') + self.assertEqual(family.coded_phenotype, 'burnt') + + await self.flayer.update_family(family_id, external_ids={'foo': 'bar'}) + family = await self.flayer.get_family_by_internal_id(family_id) + self.assertEqual(family.external_ids['foo'], 'bar') + + await self.flayer.update_family(family_id, external_ids={'foo': 'baz'}) + family = await self.flayer.get_family_by_internal_id(family_id) + self.assertEqual(family.external_ids['foo'], 'baz') + + await self.flayer.update_family(family_id, external_ids={'foo': None}) + family = await self.flayer.get_family_by_internal_id(family_id) + self.assertDictEqual(family.external_ids, {PRIMARY_EXTERNAL_ORG: 'Smith'}) + + await self.flayer.update_family(family_id, description='Goldsmiths') + family = await self.flayer.get_family_by_internal_id(family_id) + self.assertEqual(family.description, 'Goldsmiths') + self.assertEqual(family.coded_phenotype, 'burnt') + + await self.flayer.update_family(family_id, coded_phenotype='gilt') + family = await self.flayer.get_family_by_internal_id(family_id) + self.assertEqual(family.description, 'Goldsmiths') + self.assertEqual(family.coded_phenotype, 'gilt') + + @run_as_sync + async def test_bad_query(self): + """Exercise invalid query() usage""" + with self.assertRaises(ValueError): + await self.flayer.query(FamilyFilter()) + + @run_as_sync + async def test_none_by_participants(self): + """Exercise get_families_by_participants() method""" + result = await self.flayer.get_families_by_participants([]) + self.assertDictEqual(result, {}) + + @run_as_sync + async def test_import_families(self): + """Exercise import_families() method""" + await self.flayer.import_families( + ['familyid', 'description', 'phenotype'], + [ + ['Smith', 'Blacksmiths', 'burnt'], + ['Jones', 'From Wales', 'sings well'], + ['Taylor', 'Post Norman', 'sews'], + ], + ) + + result = await self.flayer.query( + FamilyFilter(project=GenericFilter(eq=self.project_id)) + ) + self.assertEqual(len(result), 3) + family = {f.external_ids[PRIMARY_EXTERNAL_ORG]: f for f in result} + self.assertEqual(family['Smith'].description, 'Blacksmiths') + self.assertEqual(family['Smith'].coded_phenotype, 'burnt') + self.assertEqual(family['Jones'].description, 'From Wales') + self.assertEqual(family['Jones'].coded_phenotype, 'sings well') + self.assertEqual(family['Taylor'].description, 'Post Norman') + self.assertEqual(family['Taylor'].coded_phenotype, 'sews') + + await self.flayer.import_families( + ['familyid', 'description', 'phenotype'], + [ + ['Smith', 'Goldsmiths actually', 'gilt'], + ['Brown', 'From Jamaica', 'brunette'], + ], + ) + + result = await self.flayer.query( + FamilyFilter(project=GenericFilter(eq=self.project_id)) + ) + self.assertEqual(len(result), 4) + family = {f.external_ids[PRIMARY_EXTERNAL_ORG]: f for f in result} + self.assertEqual(family['Smith'].description, 'Goldsmiths actually') + self.assertEqual(family['Smith'].coded_phenotype, 'gilt') + self.assertEqual(family['Brown'].description, 'From Jamaica') + self.assertEqual(family['Brown'].coded_phenotype, 'brunette') + self.assertEqual(family['Jones'].description, 'From Wales') + self.assertEqual(family['Jones'].coded_phenotype, 'sings well') + self.assertEqual(family['Taylor'].description, 'Post Norman') + self.assertEqual(family['Taylor'].coded_phenotype, 'sews') + + @run_as_sync + async def test_direct_get_id_map(self): + """Exercise the table's get_id_map_by_internal_ids() method""" + ftable = FamilyTable(self.connection) + + result = await ftable.get_id_map_by_internal_ids([]) + self.assertDictEqual(result, {}) + + result = await ftable.get_id_map_by_internal_ids([42], allow_missing=True) + self.assertDictEqual(result, {}) + + with self.assertRaises(NotFoundError): + _ = await ftable.get_id_map_by_internal_ids([42]) diff --git a/test/test_search.py b/test/test_search.py index 0f14a776a..d26fa06f6 100644 --- a/test/test_search.py +++ b/test/test_search.py @@ -188,7 +188,7 @@ async def test_search_family(self): Search family by External ID should only return one result """ - f_id = await self.flayer.create_family(external_id='FAMXX01') + f_id = await self.flayer.create_family(external_ids={'forg': 'FAMXX01'}) results = await self.schlay.search( query='FAMXX01', project_ids=[self.project_id] ) @@ -208,7 +208,7 @@ async def test_search_mixed(self): p = await self.player.upsert_participant( ParticipantUpsertInternal(external_ids={PRIMARY_EXTERNAL_ORG: 'X:PART01'}) ) - f_id = await self.flayer.create_family(external_id='X:FAM01') + f_id = await self.flayer.create_family(external_ids={'famxorg': 'X:FAM01'}) await fptable.create_rows( [ PedRowInternal( diff --git a/test/test_update_participant_family.py b/test/test_update_participant_family.py index 1774fb3c7..95c41398f 100644 --- a/test/test_update_participant_family.py +++ b/test/test_update_participant_family.py @@ -16,8 +16,10 @@ async def setUp(self) -> None: fl = FamilyLayer(self.connection) - self.fid_1 = await fl.create_family(external_id='FAM01') - self.fid_2 = await fl.create_family(external_id='FAM02') + self.fid_1 = await fl.create_family(external_ids={'forg': 'FAM01'}) + self.fid_2 = await fl.create_family(external_ids={'forg': 'FAM02'}) + # Also exercise update_family() + await fl.update_family(self.fid_2, external_ids={'otherorg': 'OFAM02'}) pl = ParticipantLayer(self.connection) self.pid = ( diff --git a/test/test_web.py b/test/test_web.py index bcca16e3a..85146d63d 100644 --- a/test/test_web.py +++ b/test/test_web.py @@ -362,7 +362,7 @@ def get_test_participant_2(): fields={ MetaSearchEntityPrefix.FAMILY: [ ProjectParticipantGridField( - key='external_id', label='Family ID', is_visible=True + key='external_ids', label='Family ID', is_visible=True ) ], MetaSearchEntityPrefix.PARTICIPANT: [ @@ -756,7 +756,7 @@ def test_nested_participant_to_rows(self): id=1, external_ids={PRIMARY_EXTERNAL_ORG: 'pex1'}, meta={'pkey': 'value'}, - families=[FamilySimple(id=-2, external_id='fex1')], + families=[FamilySimple(id=-2, external_ids={'family_org': 'fex1'})], samples=[ NestedSample( id='xpgA', @@ -800,7 +800,7 @@ def test_nested_participant_to_rows(self): fields={ MetaSearchEntityPrefix.FAMILY: [ ProjectParticipantGridField( - key='external_id', label='', is_visible=True + key='external_ids', label='', is_visible=True ) ], MetaSearchEntityPrefix.PARTICIPANT: [ @@ -840,7 +840,7 @@ def test_nested_participant_to_rows(self): self.assertTupleEqual( headers, ( - 'family.external_id', + 'family.external_ids', 'participant.external_ids', 'participant.meta.pkey', 'sample.meta.skey', @@ -852,7 +852,7 @@ def test_nested_participant_to_rows(self): ), ) non_sg_keys = ( - 'fex1', + 'family_org: fex1', 'pex1', 'value', 'svalue', diff --git a/web/src/pages/project/ParticipantGridRow.tsx b/web/src/pages/project/ParticipantGridRow.tsx index 38b144ac3..70e730f0a 100644 --- a/web/src/pages/project/ParticipantGridRow.tsx +++ b/web/src/pages/project/ParticipantGridRow.tsx @@ -83,7 +83,7 @@ const FamilyCells: React.FC<{ key={`family-${participant.id}-${f.id}`} id={`${f.id ?? ''}`} > - {f.external_id} + {prepareExternalIds(f.external_ids || {})} )) : participant.families