diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ac30f147a..4370f085f 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 7.4.1 +current_version = 7.4.2 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P[A-z0-9-]+) diff --git a/api/server.py b/api/server.py index cd8df4fee..ce94c5495 100644 --- a/api/server.py +++ b/api/server.py @@ -25,7 +25,7 @@ from db.python.utils import get_logger # This tag is automatically updated by bump2version -_VERSION = '7.4.1' +_VERSION = '7.4.2' logger = get_logger() diff --git a/db/python/layers/project_insights.py b/db/python/layers/project_insights.py index c7ec38355..0e939cc9d 100644 --- a/db/python/layers/project_insights.py +++ b/db/python/layers/project_insights.py @@ -277,6 +277,16 @@ def get_sg_web_report_links( return report_links + def get_cram_record(self, cram_row: AnalysisRow | None): + """Get the CRAM record for a sequencing group""" + return { + 'id': cram_row.id if cram_row else None, + 'output': cram_row.output if cram_row else None, + 'timestamp_completed': cram_row.timestamp_completed.strftime('%d-%m-%y') + if cram_row + else None, + } + def get_analysis_stats_internal_from_record( self, analysis_row: AnalysisRow | None, @@ -339,7 +349,7 @@ def get_insights_details_internal_row( sequencing_platform: SequencingPlatform, sequencing_technology: SequencingTechnology, sequencing_group_details: SequencingGroupDetailRow, - sequencing_groups_with_crams: list[SequencingGroupInternalId], + sequencing_group_cram: AnalysisRow, analysis_sequencing_groups: dict[AnalysisId, list[SequencingGroupInternalId]], latest_annotate_dataset_id: AnalysisId | None, latest_snv_es_index_id: AnalysisId | None, @@ -363,8 +373,8 @@ def get_insights_details_internal_row( sgs_in_latest_sv_es_index = analysis_sequencing_groups.get( latest_sv_es_index_id, [] ) + sg_cram = self.get_cram_record(sequencing_group_cram) - # participant_ext_ids = self.convert_to_external_ids(sequencing_group_details.participant_external_id) sample_ext_ids = self.convert_to_external_ids( sequencing_group_details.sample_external_ids ) @@ -382,8 +392,7 @@ def get_insights_details_internal_row( sample_id=sequencing_group_details.sample_id, sample_ext_ids=sample_ext_ids, sequencing_group_id=sequencing_group_details.sequencing_group_id, - completed_cram=sequencing_group_details.sequencing_group_id - in sequencing_groups_with_crams, + cram=sg_cram, in_latest_annotate_dataset=sequencing_group_details.sequencing_group_id in sgs_in_latest_annotate_dataset, in_latest_snv_es_index=sequencing_group_details.sequencing_group_id @@ -528,24 +537,24 @@ async def _crams_by_project_id_and_seq_fields( sequencing_types: list[SequencingType], ) -> dict[ProjectSeqTypeTechnologyKey, list[SequencingGroupInternalId]]: _query = """ - SELECT - a.project, - sg.type as sequencing_type, - sg.technology as sequencing_technology, - GROUP_CONCAT(DISTINCT asg.sequencing_group_id) as sequencing_group_ids - FROM - analysis a - LEFT JOIN analysis_sequencing_group asg ON a.id = asg.analysis_id - LEFT JOIN sequencing_group sg ON sg.id = asg.sequencing_group_id - WHERE - a.project IN :projects - AND sg.type IN :sequencing_types - AND a.type = 'CRAM' - AND a.status = 'COMPLETED' - GROUP BY - a.project, - sg.type, - sg.technology; +SELECT + a.project, + sg.type as sequencing_type, + sg.technology as sequencing_technology, + GROUP_CONCAT(DISTINCT asg.sequencing_group_id) as sequencing_group_ids +FROM + analysis a + LEFT JOIN analysis_sequencing_group asg ON a.id = asg.analysis_id + LEFT JOIN sequencing_group sg ON sg.id = asg.sequencing_group_id +WHERE + a.project IN :projects + AND sg.type IN :sequencing_types + AND a.type = 'CRAM' + AND a.status = 'COMPLETED' +GROUP BY + a.project, + sg.type, + sg.technology; """ _query_results = await self.connection.fetch_all( @@ -559,6 +568,72 @@ async def _crams_by_project_id_and_seq_fields( _query_results, 'sequencing_group_ids' ) + async def _sg_crams_by_project_id_and_seq_fields( + self, project_ids: list[ProjectId], sequencing_types: list[str] + ) -> dict[ + ProjectSeqTypeTechnologyKey, dict[SequencingGroupInternalId, AnalysisRow] + ]: + _query = """ +SELECT + a.project, + a.id as analysis_id, + sg.id as sequencing_group_id, + sg.type as sequencing_type, + sg.technology as sequencing_technology, + COALESCE(a.output, ao.output, of.path) as output, + a.timestamp_completed +FROM + analysis a + LEFT JOIN analysis_sequencing_group asg ON a.id = asg.analysis_id + LEFT JOIN analysis_outputs ao ON a.id = ao.analysis_id + LEFT JOIN output_file of ON ao.file_id = of.id + LEFT JOIN sequencing_group sg ON sg.id = asg.sequencing_group_id + INNER JOIN ( + SELECT + asg.sequencing_group_id, + MAX(a.timestamp_completed) as max_timestamp + FROM analysis a + INNER JOIN analysis_sequencing_group asg ON a.id = asg.analysis_id + WHERE a.type='CRAM' + AND a.status='COMPLETED' + AND a.project IN :projects + GROUP BY asg.sequencing_group_id + ) max_timestamps ON asg.sequencing_group_id = max_timestamps.sequencing_group_id + AND a.timestamp_completed = max_timestamps.max_timestamp +WHERE + a.project IN :projects + AND sg.type IN :sequencing_types + AND a.type = 'CRAM' + AND a.status = 'COMPLETED'; + """ + _query_results = await self.connection.fetch_all( + _query, + { + 'projects': project_ids, + 'sequencing_types': sequencing_types, + }, + ) + + cram_timestamps_by_project_id_and_seq_fields: dict[ + ProjectSeqTypeTechnologyKey, dict[SequencingGroupInternalId, AnalysisRow] + ] = {} + for row in _query_results: + key = ProjectSeqTypeTechnologyKey( + row['project'], + row['sequencing_type'], + row['sequencing_technology'], + ) + sg_id = row['sequencing_group_id'] + cram_row = AnalysisRow( + id=row['analysis_id'], + output=row['output'], + timestamp_completed=row['timestamp_completed'], + ) + if key not in cram_timestamps_by_project_id_and_seq_fields: + cram_timestamps_by_project_id_and_seq_fields[key] = {} + cram_timestamps_by_project_id_and_seq_fields[key][sg_id] = cram_row + return cram_timestamps_by_project_id_and_seq_fields + async def _latest_annotate_dataset_by_project_id_and_seq_type( self, project_ids: list[ProjectId], sequencing_types: list[str] ) -> dict[ProjectSeqTypeKey, AnalysisRow]: @@ -585,11 +660,12 @@ async def _latest_annotate_dataset_by_project_id_and_seq_type( ) max_timestamps ON a.project = max_timestamps.project AND a.timestamp_completed = max_timestamps.max_timestamp AND JSON_UNQUOTE(JSON_EXTRACT(a.meta, '$.sequencing_type')) = max_timestamps.sequencing_type -WHERE a.type = 'CUSTOM' -AND a.status = 'COMPLETED' -AND a.project IN :projects -AND JSON_UNQUOTE(JSON_EXTRACT(a.meta, '$.sequencing_type')) IN :sequencing_types -AND JSON_EXTRACT(a.meta, '$.stage') = 'AnnotateDataset'; +WHERE + a.type = 'CUSTOM' + AND a.status = 'COMPLETED' + AND a.project IN :projects + AND JSON_UNQUOTE(JSON_EXTRACT(a.meta, '$.sequencing_type')) IN :sequencing_types + AND JSON_EXTRACT(a.meta, '$.stage') = 'AnnotateDataset'; -- JSON_UNQUOTE is necessary to compare JSON values with IN operator """ _query_results = await self.connection.fetch_all( @@ -634,8 +710,9 @@ async def _latest_es_indices_by_project_id_and_seq_type_and_stage( AND a.timestamp_completed = max_timestamps.max_timestamp AND JSON_UNQUOTE(JSON_EXTRACT(a.meta, '$.sequencing_type')) = max_timestamps.sequencing_type AND JSON_EXTRACT(a.meta, '$.stage') = max_timestamps.stage -WHERE a.project IN :projects -AND JSON_UNQUOTE(JSON_EXTRACT(a.meta, '$.sequencing_type')) in :sequencing_types; +WHERE + a.project IN :projects + AND JSON_UNQUOTE(JSON_EXTRACT(a.meta, '$.sequencing_type')) in :sequencing_types; """ _query_results = await self.connection.fetch_all( _query, @@ -736,12 +813,14 @@ async def _details_stripy_reports( SELECT a.project, a.id, - a.output, + coalesce(a.output, ao.output, of.path) as output, a.timestamp_completed, asg.sequencing_group_id, - JSON_EXTRACT(meta, '$.outliers_detected') as outliers_detected, - JSON_QUERY(meta, '$.outlier_loci') as outlier_loci + JSON_EXTRACT(a.meta, '$.outliers_detected') as outliers_detected, + JSON_QUERY(a.meta, '$.outlier_loci') as outlier_loci FROM analysis a +LEFT JOIN analysis_outputs ao on a.id=ao.analysis_id +LEFT JOIN output_file of on of.id = ao.file_id LEFT JOIN analysis_sequencing_group asg on asg.analysis_id=a.id INNER JOIN ( SELECT @@ -782,10 +861,12 @@ async def _details_mito_reports( SELECT a.project, a.id, - a.output, + coalesce(a.output, ao.output, of.path) as output, a.timestamp_completed, asg.sequencing_group_id FROM analysis a +LEFT JOIN analysis_outputs ao on a.id=ao.analysis_id +LEFT JOIN output_file of on of.id = ao.file_id LEFT JOIN analysis_sequencing_group asg on asg.analysis_id=a.id INNER JOIN ( SELECT @@ -979,7 +1060,7 @@ async def get_project_insights_details( self._sequencing_group_details_by_project_and_seq_fields( project_ids, sequencing_types ), - self._crams_by_project_id_and_seq_fields(project_ids, sequencing_types), + self._sg_crams_by_project_id_and_seq_fields(project_ids, sequencing_types), self._latest_annotate_dataset_by_project_id_and_seq_type( project_ids, sequencing_types ), @@ -1021,8 +1102,10 @@ async def get_project_insights_details( ): continue - sequencing_groups_with_crams = crams_by_project_id_and_seq_fields.get( - (project.id, seq_type, seq_tech), [] + sequencing_groups_crams: dict[SequencingGroupInternalId, AnalysisRow] = ( + crams_by_project_id_and_seq_fields.get( + (project.id, seq_type, seq_tech), {} + ) ) ( latest_annotate_dataset_row, @@ -1039,6 +1122,7 @@ async def get_project_insights_details( for details_row in details_rows: if not details_row: continue + sg_id = details_row.sequencing_group_id response.append( self.get_insights_details_internal_row( project=project, @@ -1046,7 +1130,7 @@ async def get_project_insights_details( sequencing_platform=seq_platform, sequencing_technology=seq_tech, sequencing_group_details=details_row, - sequencing_groups_with_crams=sequencing_groups_with_crams, + sequencing_group_cram=sequencing_groups_crams.get(sg_id), analysis_sequencing_groups=analysis_sequencing_groups, latest_annotate_dataset_id=( latest_annotate_dataset_row.id diff --git a/deploy/python/version.txt b/deploy/python/version.txt index 815da58b7..f8cb1fa11 100644 --- a/deploy/python/version.txt +++ b/deploy/python/version.txt @@ -1 +1 @@ -7.4.1 +7.4.2 diff --git a/models/models/project_insights.py b/models/models/project_insights.py index adf6ba110..c779e2481 100644 --- a/models/models/project_insights.py +++ b/models/models/project_insights.py @@ -59,7 +59,7 @@ class ProjectInsightsDetailsInternal: sample_id: int sample_ext_ids: list[str] sequencing_group_id: int - completed_cram: bool + cram: dict[str, Any] in_latest_annotate_dataset: bool in_latest_snv_es_index: bool in_latest_sv_es_index: bool @@ -83,7 +83,7 @@ def to_external(self): sequencing_group_id=sequencing_group_id_format.sequencing_group_id_format( self.sequencing_group_id ), - completed_cram=self.completed_cram, + cram=self.cram, in_latest_annotate_dataset=self.in_latest_annotate_dataset, in_latest_snv_es_index=self.in_latest_snv_es_index, in_latest_sv_es_index=self.in_latest_sv_es_index, @@ -107,7 +107,7 @@ class ProjectInsightsDetails(SMBase): sample_id: str sample_ext_ids: list[str] sequencing_group_id: str - completed_cram: bool + cram: dict[str, Any] in_latest_annotate_dataset: bool in_latest_snv_es_index: bool in_latest_sv_es_index: bool diff --git a/setup.py b/setup.py index f511e92fa..374ab7690 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( name=PKG, # This tag is automatically updated by bump2version - version='7.4.1', + version='7.4.2', description='Python API for interacting with the Sample API system', long_description=readme, long_description_content_type='text/markdown', diff --git a/test/data/generate_seqr_project_data.py b/test/data/generate_seqr_project_data.py index 6df77002d..334ef7946 100644 --- a/test/data/generate_seqr_project_data.py +++ b/test/data/generate_seqr_project_data.py @@ -72,6 +72,16 @@ 'IJKL', ] +QUERY_PROJECT_ID = gql( + """ + query ProjectIdQuery($project: String!) { + project(name: $project) { + id + } + } + """ +) + QUERY_PROJECT_SGS = gql( """ query MyQuery($project: String!) { @@ -385,7 +395,7 @@ async def generate_sample_entries( async def generate_cram_analyses( - project: str, analyses_to_insert: list[Analysis] + project: str, project_id: int, analyses_to_insert: list[Analysis] ) -> list[dict]: """ Queries the list of sequencing groups for a project and randomly selects some @@ -409,6 +419,7 @@ async def generate_cram_analyses( sequencing_group_ids=[sg['id']], type='cram', status=AnalysisStatus('completed'), + project=project_id, output=f'FAKE://{project}/crams/{sg["id"]}.cram', timestamp_completed=( datetime.datetime.now() @@ -428,6 +439,7 @@ async def generate_cram_analyses( async def generate_web_report_analyses( project: str, + project_id: int, aligned_sequencing_groups: list[dict], analyses_to_insert: list[Analysis], ): @@ -459,6 +471,7 @@ def get_stripy_outliers(): sequencing_group_ids=[sg['id']], type='web', status=AnalysisStatus('completed'), + project=project_id, output=f'FAKE://{project}/stripy/{sg["id"]}.stripy.html', timestamp_completed=( datetime.datetime.now() @@ -477,6 +490,7 @@ def get_stripy_outliers(): sequencing_group_ids=[sg['id']], type='web', status=AnalysisStatus('completed'), + project=project_id, output=f'FAKE://{project}/mito/mitoreport-{sg["id"]}/index.html', timestamp_completed=( datetime.datetime.now() @@ -613,13 +627,22 @@ async def main(): ) logging.info(f'Set {project} as seqr project') + project_id_query_result = await query_async( + QUERY_PROJECT_ID, {'project': project} + ) + project_id = project_id_query_result['project']['id'] + participant_id_map = await generate_project_pedigree(project) await generate_sample_entries(project, participant_id_map, metamist_enums, sapi) - aligned_sgs = await generate_cram_analyses(project, analyses_to_insert) + aligned_sgs = await generate_cram_analyses( + project, project_id, analyses_to_insert + ) - await generate_web_report_analyses(project, aligned_sgs, analyses_to_insert) + await generate_web_report_analyses( + project, project_id, aligned_sgs, analyses_to_insert + ) await generate_joint_called_analyses(project, aligned_sgs, analyses_to_insert) diff --git a/web/package.json b/web/package.json index 7ea4868f6..703554a7d 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "metamist", - "version": "7.4.1", + "version": "7.4.2", "private": true, "dependencies": { "@apollo/client": "^3.11.5", diff --git a/web/src/pages/insights/DetailsTable.tsx b/web/src/pages/insights/DetailsTable.tsx index 1a49fbe70..ba9d03a9c 100644 --- a/web/src/pages/insights/DetailsTable.tsx +++ b/web/src/pages/insights/DetailsTable.tsx @@ -28,6 +28,10 @@ const getCellValue = (details: ProjectInsightsDetails, key: ColumnKey): React.Re const report = details.web_reports?.[key] return report ? Link : 'N/A' } + if (key === 'cram') { + // Return the cram timestamp_completed string if it exists, otherwise return 'N/A' + return (details.cram as { timestamp_completed?: string })?.timestamp_completed || 'N/A' + } const value = details[key as keyof ProjectInsightsDetails] diff --git a/web/src/pages/insights/FilterData.tsx b/web/src/pages/insights/FilterData.tsx index 28d307557..3b02aea7c 100644 --- a/web/src/pages/insights/FilterData.tsx +++ b/web/src/pages/insights/FilterData.tsx @@ -58,6 +58,12 @@ function useFilterData = ({ specialSelectionLabel, specialSelectionItems, }) => { - const handleSelectAll = (selectAll: boolean) => { - onSelectionChange(selectAll ? [...items] : []) + const handleChange = (data: DropdownProps) => { + const value = data.value as string[] | undefined + if (value) { + onSelectionChange(value) + } } const handleSpecialSelection = () => { @@ -46,57 +49,33 @@ const Selector: React.FC = ({ } } - const handleItemChange = (item: string, isChecked: boolean) => { - if (isChecked) { - onSelectionChange([...selectedItems, item]) - } else { - onSelectionChange(selectedItems.filter((i) => i !== item)) - } - } - - const isSpecialSelectionChecked = specialSelectionItems - ? specialSelectionItems.every((item) => selectedItems.includes(item)) - : false + const options = items.map((item) => ({ + key: item, + text: item, + value: item, + })) return (

{title}

-
+ + {specialSelectionLabel && ( handleSelectAll(data.checked ?? false)} + label={specialSelectionLabel} + checked={specialSelectionItems?.every((item) => selectedItems.includes(item))} + onChange={handleSpecialSelection} /> - {specialSelectionLabel && ( - - )} -
-
- {items.map((item) => ( -
- handleItemChange(item, data.checked ?? false)} - /> -
- ))} -
+ )}
) } @@ -115,13 +94,13 @@ const ProjectAndSeqTypeSelector: React.FC = ({ items={projects} selectedItems={selectedProjects} onSelectionChange={onProjectChange} - title="Select Projects" + title="Projects" />