diff --git a/src/__test__/components/data-exploration/generic-gene-table/ComponentActions.test.jsx b/src/__test__/components/data-exploration/generic-gene-table/ComponentActions.test.jsx index f9aecd12ed..ca981c17f7 100644 --- a/src/__test__/components/data-exploration/generic-gene-table/ComponentActions.test.jsx +++ b/src/__test__/components/data-exploration/generic-gene-table/ComponentActions.test.jsx @@ -2,11 +2,9 @@ import React from 'react'; import { mount } from 'enzyme'; import { Provider } from 'react-redux'; import { Dropdown } from 'antd'; -import waitForActions from 'redux-mock-store-await-actions'; import thunk from 'redux-thunk'; import preloadAll from 'jest-next-dynamic'; import configureMockStore from 'redux-mock-store'; -import { GENES_EXPRESSION_LOADING, GENES_EXPRESSION_LOADED } from 'redux/actionTypes/genes'; import { UPDATE_CONFIG } from 'redux/actionTypes/componentConfig'; import fetchWork from 'utils/work/fetchWork'; import ComponentActions from 'components/data-exploration/generic-gene-table/ComponentActions'; @@ -107,7 +105,7 @@ describe('ComponentActions', () => { expect(component.find(Dropdown).length).toEqual(0); }); - it('Dispatches loadGeneExpression action with the right list of genes when Add is clicked', async () => { + it('Updates selectedGenes config when Add is clicked', () => { const store = mockStore({ ...initialState, genes: { @@ -125,28 +123,15 @@ describe('ComponentActions', () => { const menuButtons = component.find(Dropdown).props().overlay; menuButtons.props.children[0].props.onClick(); - // Wait for side-effect to propagate (config update, then genes loading and loaded). - await waitForActions(store, [UPDATE_CONFIG, GENES_EXPRESSION_LOADING, GENES_EXPRESSION_LOADED]); - - expect(fetchWork).toHaveBeenCalledWith( - experimentId, - { - name: 'GeneExpression', - genes: ['GeneA', 'GeneB', 'GeneC'], - }, - store.getState, - expect.any(Function), - { timeout: 60 }, + expect(fetchWork).not.toHaveBeenCalled(); + expect(store.getActions().length).toEqual(1); + expect(store.getActions()[0].type).toEqual(UPDATE_CONFIG); + expect(store.getActions()[0].payload.configChanges.selectedGenes).toEqual( + ['Gzma', 'Lyz2', 'GeneA', 'GeneB', 'GeneC'], ); - - expect(store.getActions().length).toEqual(3); - - expect(store.getActions()[1]).toMatchSnapshot(); - - expect(store.getActions()[2]).toMatchSnapshot(); }); - it('Dispatches loadGeneExpression action with the right list of genes when Remove is clicked', async () => { + it('Updates selectedGenes config when Remove is clicked', () => { const store = mockStore({ ...initialState, genes: { @@ -164,17 +149,13 @@ describe('ComponentActions', () => { const menuButtons = component.find(Dropdown).props().overlay; menuButtons.props.children[1].props.onClick(); - // Wait for side-effect to propagate (config update and genes loaded). - await waitForActions(store, [UPDATE_CONFIG, GENES_EXPRESSION_LOADED]); - - expect(fetchWork).toHaveBeenCalledTimes(0); - - expect(store.getActions().length).toEqual(2); - - expect(store.getActions()[1]).toMatchSnapshot(); + expect(fetchWork).not.toHaveBeenCalled(); + expect(store.getActions().length).toEqual(1); + expect(store.getActions()[0].type).toEqual(UPDATE_CONFIG); + expect(store.getActions()[0].payload.configChanges.selectedGenes).toEqual(['Lyz2']); }); - it('Dispatches loadGeneExpression action with the right list of genes when Overwrite is clicked', async () => { + it('Updates selectedGenes config when Overwrite is clicked', () => { const store = mockStore({ ...initialState, genes: { @@ -192,12 +173,9 @@ describe('ComponentActions', () => { const menuButtons = component.find(Dropdown).props().overlay; menuButtons.props.children[2].props.onClick(); - // Wait for side-effect to propagate (config update and genes loaded). - await waitForActions(store, [UPDATE_CONFIG, GENES_EXPRESSION_LOADED]); - - expect(fetchWork).toHaveBeenCalledTimes(0); - - expect(store.getActions().length).toEqual(2); - expect(store.getActions()[1]).toMatchSnapshot(); + expect(fetchWork).not.toHaveBeenCalled(); + expect(store.getActions().length).toEqual(1); + expect(store.getActions()[0].type).toEqual(UPDATE_CONFIG); + expect(store.getActions()[0].payload.configChanges.selectedGenes).toEqual(['Gzma']); }); }); diff --git a/src/__test__/components/data-exploration/generic-gene-table/__snapshots__/ComponentActions.test.jsx.snap b/src/__test__/components/data-exploration/generic-gene-table/__snapshots__/ComponentActions.test.jsx.snap deleted file mode 100644 index 8ea6bdfc9d..0000000000 --- a/src/__test__/components/data-exploration/generic-gene-table/__snapshots__/ComponentActions.test.jsx.snap +++ /dev/null @@ -1,117 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`ComponentActions Dispatches loadGeneExpression action with the right list of genes when Add is clicked 1`] = ` -{ - "payload": { - "componentUuid": "asd", - "experimentId": "1234", - "genes": [ - "Gzma", - "Lyz2", - "GeneA", - "GeneB", - "GeneC", - ], - }, - "type": "genes/expressionLoading", -} -`; - -exports[`ComponentActions Dispatches loadGeneExpression action with the right list of genes when Add is clicked 2`] = ` -{ - "payload": { - "componentUuid": "asd", - "genes": [ - "Gzma", - "Lyz2", - "GeneA", - "GeneB", - "GeneC", - ], - "newGenes": { - "orderedGeneNames": [ - "GeneA", - "GeneB", - "GeneC", - ], - "rawExpression": { - "datatype": undefined, - "index": [ - 0, - 0, - 2, - 0, - 2, - ], - "mathjs": "SparseMatrix", - "ptr": [ - 0, - 1, - 3, - 5, - ], - "size": [ - 3, - 3, - ], - "values": [ - 1, - 2, - 5, - 1, - 6, - ], - }, - "stats": { - "rawMean": [ - 0.3, - 2.3, - 1.3, - ], - "rawStdev": [ - 0.4, - 2, - 1, - ], - "truncatedMax": [ - 1, - 3, - 5, - ], - "truncatedMin": [ - 0, - 2, - 1, - ], - }, - }, - }, - "type": "genes/expressionLoaded", -} -`; - -exports[`ComponentActions Dispatches loadGeneExpression action with the right list of genes when Overwrite is clicked 1`] = ` -{ - "payload": { - "componentUuid": "asd", - "experimentId": "1234", - "genes": [ - "Gzma", - ], - }, - "type": "genes/expressionLoaded", -} -`; - -exports[`ComponentActions Dispatches loadGeneExpression action with the right list of genes when Remove is clicked 1`] = ` -{ - "payload": { - "componentUuid": "asd", - "experimentId": "1234", - "genes": [ - "Lyz2", - ], - }, - "type": "genes/expressionLoaded", -} -`; diff --git a/src/__test__/components/data-exploration/heatmap/__snapshots__/HeatmapPlot.test.jsx.snap b/src/__test__/components/data-exploration/heatmap/__snapshots__/HeatmapPlot.test.jsx.snap index c3ce5d3ac3..131c33666c 100644 --- a/src/__test__/components/data-exploration/heatmap/__snapshots__/HeatmapPlot.test.jsx.snap +++ b/src/__test__/components/data-exploration/heatmap/__snapshots__/HeatmapPlot.test.jsx.snap @@ -447,84 +447,83 @@ exports[`HeatmapPlot Reacts to cellClass groupby being changed 2`] = ` exports[`HeatmapPlot Renders the heatmap component by default if everything loads 1`] = ` Uint8Array [ + 175, 0, - 74, 0, - 93, 0, + 135, 0, - 255, - 255, 0, - 126, 0, 0, 0, - 255, + 219, 0, 0, 0, + 103, + 253, 0, - 112, - 80, + 119, 0, + 205, 0, - 103, 0, - 206, - 197, 0, + 119, 0, + 254, 0, + 108, 0, + 218, 0, 0, 0, - 119, 0, 0, 0, 0, 0, + 255, 0, 0, - 188, 0, 0, + 158, 0, - 208, 0, 0, 0, - 199, - 219, + 123, + 88, 0, 0, 0, - 103, - 175, 0, 0, 0, - 135, 0, - 255, - 255, 0, - 188, 0, 0, + 181, 0, + 155, 0, + 255, 0, 0, + 103, 0, - 54, - 64, + 206, 0, + 74, 0, + 93, 0, 0, + 188, 0, 0, 0, @@ -535,168 +534,164 @@ Uint8Array [ 0, 0, 0, - 158, 0, 0, 0, + 197, + 0, 0, - 227, - 87, 0, 0, 0, 0, + 54, + 64, + 0, + 0, + 255, + 255, 0, + 188, 245, 0, 97, 0, 213, - 181, 0, - 155, 0, - 255, - 253, 0, - 119, 0, - 205, 0, 0, 0, 0, + 112, + 80, 0, - 254, 0, - 108, 0, - 218, + 227, + 87, 0, + 255, + 255, 0, + 126, + 208, 0, - 123, - 88, 0, 0, + 199, 0, 0, + 88, + 104, 0, 0, + 168, + 92, 0, - 95, - 162, 0, 0, 0, - 88, - 104, 0, 0, 0, 0, 0, 0, + 135, 0, 0, 0, - 227, + 95, + 162, 0, + 129, 0, + 138, 0, - 90, + 170, 0, - 221, 0, 0, - 122, - 0, + 174, 0, + 159, 0, - 168, - 92, 0, 0, + 153, 133, 0, 142, 0, 237, 0, - 216, - 0, - 127, - 0, - 0, 0, 161, 0, 203, - 151, 0, 0, - 136, - 97, - 0, - 0, - 0, - 174, + 90, 0, + 221, 0, 0, - 77, + 119, + 194, 0, + 125, + 127, 0, + 112, 0, + 151, 0, - 119, - 194, 0, - 189, + 136, + 97, 0, - 169, 0, - 250, - 129, + 130, 0, - 138, 0, - 170, 0, 0, + 89, 0, - 135, 0, 0, 0, + 77, 0, 0, 0, - 125, - 127, 0, - 112, 0, - 159, 0, 0, 0, - 153, 0, + 122, 0, - 130, 0, + 189, 0, + 169, 0, + 250, 0, - 89, + 216, 0, + 127, 0, - 239, 0, 0, 0, 227, + 0, 232, 0, 0, @@ -707,6 +702,11 @@ Uint8Array [ 0, 0, 255, + 239, + 0, + 0, + 0, + 227, 0, 0, 0, @@ -737,11 +737,6 @@ Uint8Array [ 0, 0, 202, - 0, - 0, - 97, - 196, - 0, 254, 0, 190, @@ -749,6 +744,11 @@ Uint8Array [ 146, 0, 0, + 97, + 196, + 0, + 0, + 0, 0, 0, 168, @@ -777,25 +777,25 @@ Uint8Array [ 73, 86, 0, + 145, 0, 0, - 85, - 0, - 198, + 181, + 170, 254, 0, 102, 122, 248, - 145, 0, 0, - 181, - 170, + 85, 0, + 198, 0, 0, - 218, + 94, + 161, 0, 130, 0, @@ -809,29 +809,24 @@ Uint8Array [ 0, 0, 0, - 94, - 161, - 0, - 131, - 0, - 98, - 117, - 247, 0, + 218, 0, 0, 0, 0, 0, 0, + 131, 0, + 98, + 117, + 247, 0, - 225, 0, - 138, - 147, 0, 0, + 225, 206, 0, 89, @@ -842,16 +837,21 @@ Uint8Array [ 161, 0, 203, - 184, 0, + 138, + 147, 0, 0, - 118, 254, 0, 148, 0, 208, + 184, + 0, + 0, + 0, + 118, 0, 0, 109, @@ -862,16 +862,16 @@ Uint8Array [ 173, 0, 0, + 207, 0, 0, 0, - 255, + 251, 0, - 207, 0, 0, + 255, 0, - 251, 0, 0, 104, @@ -892,83 +892,83 @@ exports[`HeatmapPlot Renders the heatmap component by default if everything load exports[`HeatmapPlot Responds correctly to vitessce Heatmap callbacks 1`] = ` Uint8Array [ + 175, 0, - 74, 0, - 93, 0, + 135, 0, - 255, - 255, 0, - 126, 0, 0, 0, - 255, + 219, 0, 0, 0, + 103, + 253, 0, - 112, - 80, + 119, 0, + 205, 0, - 103, 0, - 206, - 197, 0, + 119, 0, + 254, 0, + 108, 0, + 218, 0, 0, 0, - 119, 0, 0, 0, 0, 0, + 255, 0, 0, - 188, 0, 0, + 158, 0, - 208, 0, 0, 0, - 199, - 219, + 123, + 88, 0, 0, 0, - 103, - 175, 0, 0, 0, - 135, 0, - 255, - 255, 0, - 188, 0, 0, + 181, + 0, + 155, 0, + 255, 0, 0, + 103, 0, + 206, 0, - 54, - 64, + 74, 0, + 93, 0, 0, + 188, 0, 0, 0, @@ -980,62 +980,64 @@ Uint8Array [ 0, 0, 0, - 158, 0, 0, + 197, 0, 0, - 227, - 87, 0, 0, 0, 0, + 54, + 64, + 0, + 0, + 255, + 255, 0, + 188, 245, 0, 97, 0, 213, - 181, - 0, - 155, 0, - 255, - 253, 0, - 119, 0, - 205, 0, 0, 0, 0, 0, - 254, + 112, + 80, 0, - 108, 0, - 218, 0, + 227, + 87, 0, + 255, + 255, 0, - 123, - 88, + 126, + 208, 0, 0, 0, + 199, 0, 0, + 88, + 104, 0, 0, - 95, - 162, + 168, + 92, 0, 0, 0, - 88, - 104, 0, 0, 0, @@ -1043,105 +1045,98 @@ Uint8Array [ 0, 0, 0, + 135, 0, 0, - 227, 0, + 95, + 162, 0, + 129, 0, - 90, + 138, 0, - 221, + 170, 0, 0, - 122, 0, + 174, 0, + 159, 0, - 168, - 92, 0, 0, + 153, 133, 0, 142, 0, 237, 0, - 216, - 0, - 127, - 0, - 0, 0, 161, 0, 203, - 151, - 0, - 0, - 136, - 97, - 0, 0, 0, - 174, + 90, 0, + 221, 0, 0, - 77, + 119, + 194, 0, + 125, + 127, 0, + 112, 0, + 151, 0, - 119, - 194, 0, - 189, + 136, + 97, 0, - 169, 0, - 250, - 129, + 130, 0, - 138, 0, - 170, 0, 0, + 89, 0, - 135, 0, 0, 0, + 77, 0, 0, 0, - 125, - 127, 0, - 112, 0, - 159, 0, 0, 0, - 153, 0, + 122, 0, - 130, 0, + 189, 0, + 169, 0, + 250, 0, - 89, + 216, 0, + 127, 0, - 239, 0, 0, 0, 227, + 0, 232, 0, 0, @@ -1152,6 +1147,11 @@ Uint8Array [ 0, 0, 255, + 239, + 0, + 0, + 0, + 227, 0, 0, 0, @@ -1182,11 +1182,6 @@ Uint8Array [ 0, 0, 202, - 0, - 0, - 97, - 196, - 0, 254, 0, 190, @@ -1194,6 +1189,11 @@ Uint8Array [ 146, 0, 0, + 97, + 196, + 0, + 0, + 0, 0, 0, 168, @@ -1222,25 +1222,25 @@ Uint8Array [ 73, 86, 0, + 145, 0, 0, - 85, - 0, - 198, + 181, + 170, 254, 0, 102, 122, 248, - 145, 0, 0, - 181, - 170, + 85, 0, + 198, 0, 0, - 218, + 94, + 161, 0, 130, 0, @@ -1254,29 +1254,24 @@ Uint8Array [ 0, 0, 0, - 94, - 161, - 0, - 131, - 0, - 98, - 117, - 247, 0, + 218, 0, 0, 0, 0, 0, 0, + 131, 0, + 98, + 117, + 247, 0, - 225, 0, - 138, - 147, 0, 0, + 225, 206, 0, 89, @@ -1287,16 +1282,21 @@ Uint8Array [ 161, 0, 203, - 184, 0, + 138, + 147, 0, 0, - 118, 254, 0, 148, 0, 208, + 184, + 0, + 0, + 0, + 118, 0, 0, 109, @@ -1307,16 +1307,16 @@ Uint8Array [ 173, 0, 0, + 207, 0, 0, 0, - 255, + 251, 0, - 207, 0, 0, + 255, 0, - 251, 0, 0, 104, @@ -1337,82 +1337,83 @@ exports[`HeatmapPlot Responds correctly to vitessce Heatmap callbacks 2`] = ` exports[`HeatmapPlot Shows loader message if the marker genes are loaded but there's other selected genes still loading 1`] = ` Uint8Array [ + 175, 0, - 74, 0, - 93, 0, + 135, 0, - 255, - 255, 0, - 126, 0, 0, 0, - 255, + 219, 0, 0, 0, + 103, + 253, 0, - 112, - 80, + 119, 0, + 205, 0, - 103, 0, - 206, - 197, 0, + 119, + 0, + 254, + 0, + 108, 0, + 218, 0, 0, 0, 0, 0, - 119, 0, 0, 0, + 255, 0, 0, 0, 0, - 188, + 158, 0, 0, 0, - 208, 0, + 123, + 88, 0, 0, - 199, - 219, 0, 0, 0, - 103, - 175, 0, 0, 0, - 135, 0, - 255, - 255, 0, - 188, + 181, 0, + 155, 0, + 255, 0, 0, + 103, 0, + 206, 0, + 74, 0, - 54, - 64, + 93, 0, 0, + 188, 0, 0, 0, @@ -1425,62 +1426,63 @@ Uint8Array [ 0, 0, 0, - 158, + 0, + 197, 0, 0, 0, 0, - 227, - 87, 0, 0, + 54, + 64, 0, 0, + 255, + 255, 0, + 188, 245, 0, 97, 0, 213, - 181, - 0, - 155, 0, - 255, - 253, 0, - 119, 0, - 205, 0, 0, 0, 0, 0, - 254, + 112, + 80, 0, - 108, 0, - 218, 0, + 227, + 87, 0, + 255, + 255, 0, - 123, - 88, + 126, + 208, 0, 0, 0, + 199, 0, 0, + 88, + 104, 0, 0, - 95, - 162, + 168, + 92, 0, 0, 0, - 88, - 104, 0, 0, 0, @@ -1488,105 +1490,98 @@ Uint8Array [ 0, 0, 0, + 135, 0, 0, - 227, 0, + 95, + 162, 0, + 129, 0, - 90, + 138, 0, - 221, + 170, 0, 0, - 122, 0, + 174, 0, + 159, 0, - 168, - 92, 0, 0, + 153, 133, 0, 142, 0, 237, 0, - 216, - 0, - 127, - 0, - 0, 0, 161, 0, 203, - 151, - 0, - 0, - 136, - 97, - 0, 0, 0, - 174, + 90, 0, + 221, 0, 0, - 77, + 119, + 194, 0, + 125, + 127, 0, + 112, 0, + 151, 0, - 119, - 194, 0, - 189, + 136, + 97, 0, - 169, 0, - 250, - 129, + 130, 0, - 138, 0, - 170, 0, 0, + 89, 0, - 135, 0, 0, 0, + 77, 0, 0, 0, - 125, - 127, 0, - 112, 0, - 159, 0, 0, 0, - 153, 0, + 122, 0, - 130, 0, + 189, 0, + 169, 0, + 250, 0, - 89, + 216, 0, + 127, 0, - 239, 0, 0, 0, 227, + 0, 232, 0, 0, @@ -1597,6 +1592,11 @@ Uint8Array [ 0, 0, 255, + 239, + 0, + 0, + 0, + 227, 0, 0, 0, @@ -1627,11 +1627,6 @@ Uint8Array [ 0, 0, 202, - 0, - 0, - 97, - 196, - 0, 254, 0, 190, @@ -1639,6 +1634,11 @@ Uint8Array [ 146, 0, 0, + 97, + 196, + 0, + 0, + 0, 0, 0, 168, @@ -1667,25 +1667,25 @@ Uint8Array [ 73, 86, 0, + 145, 0, 0, - 85, - 0, - 198, + 181, + 170, 254, 0, 102, 122, 248, - 145, 0, 0, - 181, - 170, + 85, 0, + 198, 0, 0, - 218, + 94, + 161, 0, 130, 0, @@ -1699,29 +1699,24 @@ Uint8Array [ 0, 0, 0, - 94, - 161, - 0, - 131, - 0, - 98, - 117, - 247, 0, + 218, 0, 0, 0, 0, 0, 0, + 131, 0, + 98, + 117, + 247, 0, - 225, 0, - 138, - 147, 0, 0, + 225, 206, 0, 89, @@ -1732,16 +1727,21 @@ Uint8Array [ 161, 0, 203, - 184, 0, + 138, + 147, 0, 0, - 118, 254, 0, 148, 0, 208, + 184, + 0, + 0, + 0, + 118, 0, 0, 109, @@ -1752,16 +1752,16 @@ Uint8Array [ 173, 0, 0, + 207, 0, 0, 0, - 255, + 251, 0, - 207, 0, 0, + 255, 0, - 251, 0, 0, 104, diff --git a/src/__test__/pages/__snapshots__/_error.test.jsx.snap b/src/__test__/pages/__snapshots__/_error.test.jsx.snap index 7c235e5321..633a695278 100644 --- a/src/__test__/pages/__snapshots__/_error.test.jsx.snap +++ b/src/__test__/pages/__snapshots__/_error.test.jsx.snap @@ -109,6 +109,36 @@ exports[`ErrorPage Should post error to Slack if environment is production 1`] = }, "genes": { "expression": { + "downsampled": { + "cellIds": [], + "downsampleType": null, + "error": false, + "lastFetchSettings": null, + "loading": false, + "matrix": ExpressionMatrix { + "geneIndexes": {}, + "rawGeneExpressions": { + "datatype": undefined, + "index": [], + "mathjs": "SparseMatrix", + "ptr": [ + 0, + ], + "size": [ + 0, + 0, + ], + "values": [], + }, + "setGeneExpression": [Function], + "stats": { + "rawMean": [], + "rawStdev": [], + "truncatedMax": [], + "truncatedMin": [], + }, + }, + }, "full": { "ETag": null, "error": false, @@ -299,6 +329,36 @@ exports[`ErrorPage Should post error to Slack if environment is staging 1`] = ` }, "genes": { "expression": { + "downsampled": { + "cellIds": [], + "downsampleType": null, + "error": false, + "lastFetchSettings": null, + "loading": false, + "matrix": ExpressionMatrix { + "geneIndexes": {}, + "rawGeneExpressions": { + "datatype": undefined, + "index": [], + "mathjs": "SparseMatrix", + "ptr": [ + 0, + ], + "size": [ + 0, + 0, + ], + "values": [], + }, + "setGeneExpression": [Function], + "stats": { + "rawMean": [], + "rawStdev": [], + "truncatedMax": [], + "truncatedMin": [], + }, + }, + }, "full": { "ETag": null, "error": false, diff --git a/src/__test__/pages/experiments/[experimentId]/plots-and-tables/marker-heatmap/index.test.jsx b/src/__test__/pages/experiments/[experimentId]/plots-and-tables/marker-heatmap/index.test.jsx index 28797e773e..137d85b441 100644 --- a/src/__test__/pages/experiments/[experimentId]/plots-and-tables/marker-heatmap/index.test.jsx +++ b/src/__test__/pages/experiments/[experimentId]/plots-and-tables/marker-heatmap/index.test.jsx @@ -683,9 +683,9 @@ describe('Marker heatmap plot', () => { userEvent.click(clearAllButton); }); - // Verify selectedGenes is cleared from genes.expression.views + // Verify selectedGenes is cleared from the plot config await waitFor(() => { - const selectedGenes = storeState.getState().genes.expression.views[plotUuid]?.data; + const selectedGenes = storeState.getState().componentConfig[plotUuid]?.config?.selectedGenes; expect(selectedGenes).toEqual([]); }, { timeout: 5000 }); diff --git a/src/__test__/redux/actions/experiments/__snapshots__/switchExperiment.test.js.snap b/src/__test__/redux/actions/experiments/__snapshots__/switchExperiment.test.js.snap index 3743e91aac..18b18d83db 100644 --- a/src/__test__/redux/actions/experiments/__snapshots__/switchExperiment.test.js.snap +++ b/src/__test__/redux/actions/experiments/__snapshots__/switchExperiment.test.js.snap @@ -172,6 +172,36 @@ exports[`switch experiment Then updates the experiment info 1`] = ` }, "genes": { "expression": { + "downsampled": { + "cellIds": [], + "downsampleType": null, + "error": false, + "lastFetchSettings": null, + "loading": false, + "matrix": ExpressionMatrix { + "geneIndexes": {}, + "rawGeneExpressions": { + "datatype": undefined, + "index": [], + "mathjs": "SparseMatrix", + "ptr": [ + 0, + ], + "size": [ + 0, + 0, + ], + "values": [], + }, + "setGeneExpression": [Function], + "stats": { + "rawMean": [], + "rawStdev": [], + "truncatedMax": [], + "truncatedMin": [], + }, + }, + }, "full": { "ETag": null, "error": false, @@ -523,6 +553,36 @@ exports[`switch experiment switches the experiment to its initial values 1`] = }, "genes": { "expression": { + "downsampled": { + "cellIds": [], + "downsampleType": null, + "error": false, + "lastFetchSettings": null, + "loading": false, + "matrix": ExpressionMatrix { + "geneIndexes": {}, + "rawGeneExpressions": { + "datatype": undefined, + "index": [], + "mathjs": "SparseMatrix", + "ptr": [ + 0, + ], + "size": [ + 0, + 0, + ], + "values": [], + }, + "setGeneExpression": [Function], + "stats": { + "rawMean": [], + "rawStdev": [], + "truncatedMax": [], + "truncatedMin": [], + }, + }, + }, "full": { "ETag": null, "error": false, diff --git a/src/__test__/redux/actions/genes/loadHeatmapExpression.test.js b/src/__test__/redux/actions/genes/loadHeatmapExpression.test.js new file mode 100644 index 0000000000..cb4929681c --- /dev/null +++ b/src/__test__/redux/actions/genes/loadHeatmapExpression.test.js @@ -0,0 +1,346 @@ +import configureStore from 'redux-mock-store'; +import thunk from 'redux-thunk'; +import { SparseMatrix } from 'mathjs'; + +import loadHeatmapExpression, { LARGE_DATASET_THRESHOLD } from 'redux/actions/genes/loadHeatmapExpression'; +import getInitialState from 'redux/reducers/genes/getInitialState'; + +import { + HEATMAP_EXPRESSION_LOADING, + HEATMAP_EXPRESSION_LOADED, + HEATMAP_EXPRESSION_ERROR, +} from 'redux/actionTypes/genes'; + +import fetchWork from 'utils/work/fetchWork'; + +jest.mock('utils/work/fetchWork'); + +jest.mock('utils/getTimeoutForWorkerTask', () => ({ + __esModule: true, + default: () => 60, +})); + +jest.mock('redux/actions/genes/loadGeneExpression', () => jest.fn( + () => ({ type: 'GENES_EXPRESSION_LOADING' }), +)); + +// Partial mock so we can spy on getBuckets and control the default export +jest.mock('utils/work/getHeatmapCellOrder', () => { + const actual = jest.requireActual('utils/work/getHeatmapCellOrder'); + return { + __esModule: true, + ...actual, + getBuckets: jest.fn(actual.getBuckets), + default: jest.fn(actual.default), + }; +}); + +const { getBuckets } = require('utils/work/getHeatmapCellOrder'); +const getHeatmapCellOrder = require('utils/work/getHeatmapCellOrder').default; +const loadGeneExpression = require('redux/actions/genes/loadGeneExpression'); + +const mockStore = configureStore([thunk]); + +// ─── shared mock data ──────────────────────────────────────────────────────── + +const experimentId = 'test-experiment-1'; + +// Small dataset cell sets (total cells: 3, below threshold) +const smallCellSets = { + hierarchy: [ + { key: 'louvain', children: [{ key: 'louvain-0' }] }, + { key: 'sample', children: [{ key: 'sample-1' }] }, + ], + properties: { + 'louvain-0': { cellIds: new Set([0, 1, 2]) }, + 'sample-1': { cellIds: { size: 3 } }, // only .size is used by getTotalCells + }, + loading: false, + initialLoadPending: false, + updatingClustering: false, + error: false, + hidden: new Set(), + selected: [], +}; + +// Large dataset cell sets (total cells: >= LARGE_DATASET_THRESHOLD) +const LARGE_CELL_COUNT = LARGE_DATASET_THRESHOLD + 1; +const largeCellSets = { + hierarchy: [ + { key: 'louvain', children: [{ key: 'louvain-0' }] }, + { key: 'sample', children: [{ key: 'sample-1' }] }, + ], + properties: { + 'louvain-0': { cellIds: { size: LARGE_CELL_COUNT, has: () => true, forEach: () => { } } }, + 'sample-1': { cellIds: { size: LARGE_CELL_COUNT } }, + }, + loading: false, + initialLoadPending: false, + updatingClustering: false, + error: false, + hidden: new Set(), + selected: [], +}; + +const backendStatus = { + [experimentId]: { + status: { + pipeline: { + status: 'SUCCEEDED', + startDate: '2021-01-01T01:01:01.000Z', + }, + }, + }, +}; + +const makeWorkerResponse = (overrides = {}) => ({ + orderedGeneNames: ['GENE1'], + rawExpression: { + mathjs: 'SparseMatrix', index: [], ptr: [0, 0], size: [3, 1], values: [], + }, + stats: { + rawMean: [0], rawStdev: [0], truncatedMin: [0], truncatedMax: [0], + }, + cellIds: [0, 1, 2], + ...overrides, +}); + +// ─── helpers ───────────────────────────────────────────────────────────────── + +const makeState = (cellSets, genesOverrides = {}) => { + const genesInitial = getInitialState(); + return { + genes: { + ...genesInitial, + expression: { + ...genesInitial.expression, + ...genesOverrides, + }, + }, + cellSets, + backendStatus, + }; +}; + +// ─── tests ─────────────────────────────────────────────────────────────────── + +describe('loadHeatmapExpression', () => { + const options = { + selectedCellSet: 'louvain', + groupedTracks: ['sample'], + hiddenCellSets: [], + plotUuid: 'test-plot-uuid', + }; + + beforeEach(() => { + jest.resetAllMocks(); + loadGeneExpression.mockImplementation(() => ({ type: 'GENES_EXPRESSION_LOADING' })); + }); + + it('returns null and dispatches nothing for empty genes array', async () => { + const store = mockStore(makeState(smallCellSets)); + const result = await store.dispatch(loadHeatmapExpression(experimentId, [], options)); + expect(result).toBeNull(); + expect(store.getActions()).toHaveLength(0); + }); + + it('returns null and dispatches nothing when cellSets are not accessible', async () => { + const inaccessibleCellSets = { + ...smallCellSets, + loading: true, // causes accessible=false + }; + const store = mockStore(makeState(inaccessibleCellSets)); + const result = await store.dispatch( + loadHeatmapExpression(experimentId, ['GENE1'], options), + ); + expect(result).toBeNull(); + expect(store.getActions()).toHaveLength(0); + }); + + it('delegates to loadGeneExpression for small datasets', async () => { + const store = mockStore(makeState(smallCellSets)); + await store.dispatch(loadHeatmapExpression(experimentId, ['GENE1'], options)); + expect(loadGeneExpression).toHaveBeenCalledWith(experimentId, ['GENE1'], options.plotUuid); + // loadGeneExpression mock returns a plain action object (not a thunk), so it gets dispatched + const actions = store.getActions(); + expect(actions.some((a) => a.type === 'GENES_EXPRESSION_LOADING')).toBe(true); + }); + + it('returns null and dispatches nothing if downsampled is already loading', async () => { + const state = makeState(largeCellSets, { + downsampled: { loading: true }, + }); + getBuckets.mockReturnValueOnce({ + buckets: [new Set([0, 1])], + totalSize: 2, + }); + const store = mockStore(state); + const result = await store.dispatch( + loadHeatmapExpression(experimentId, ['GENE1'], options), + ); + expect(result).toBeNull(); + expect(store.getActions()).toHaveLength(0); + }); + + it('dispatches LOADING and LOADED for large dataset (bucketed path)', async () => { + // Bucketed: cappedTotal < LARGE_DATASET_THRESHOLD + // Two small buckets → cappedTotal = 2 < 50000 + getBuckets.mockReturnValueOnce({ + buckets: [new Set([0]), new Set([1])], + totalSize: 2, + }); + fetchWork.mockResolvedValueOnce(makeWorkerResponse()); + + const store = mockStore(makeState(largeCellSets)); + await store.dispatch(loadHeatmapExpression(experimentId, ['GENE1'], options)); + + const types = store.getActions().map((a) => a.type); + expect(types).toContain(HEATMAP_EXPRESSION_LOADING); + expect(types).toContain(HEATMAP_EXPRESSION_LOADED); + }); + + it('sends bucketed downsampleSettings (selectedCellSet + groupedTracks) for bucketed path', async () => { + getBuckets.mockReturnValueOnce({ + buckets: [new Set([0]), new Set([1])], + totalSize: 2, + }); + fetchWork.mockResolvedValueOnce(makeWorkerResponse()); + + const store = mockStore(makeState(largeCellSets)); + await store.dispatch(loadHeatmapExpression(experimentId, ['GENE1'], options)); + + const [, body] = fetchWork.mock.calls[0]; + expect(body.downsampleSettings).toMatchObject({ + selectedCellSet: options.selectedCellSet, + groupedTracks: options.groupedTracks, + }); + expect(body.downsampleSettings.cellIds).toBeUndefined(); + }); + + it('dispatches LOADING and LOADED for large dataset (precomputed path)', async () => { + // Precomputed: cappedTotal = n_buckets * min(bucket.size, 1000) >= LARGE_DATASET_THRESHOLD + // 60 buckets of size 1000 → cappedTotal = 60000 >= 50000 + getBuckets.mockReturnValueOnce({ + buckets: Array.from({ length: 60 }, () => ({ size: 1000, forEach: () => { } })), + totalSize: 60000, + }); + getHeatmapCellOrder.mockReturnValueOnce([0, 1, 2]); + fetchWork.mockResolvedValueOnce(makeWorkerResponse({ cellIds: [0, 1, 2] })); + + const store = mockStore(makeState(largeCellSets)); + await store.dispatch(loadHeatmapExpression(experimentId, ['GENE1'], options)); + + const types = store.getActions().map((a) => a.type); + expect(types).toContain(HEATMAP_EXPRESSION_LOADING); + expect(types).toContain(HEATMAP_EXPRESSION_LOADED); + }); + + it('sends cellIds in downsampleSettings for precomputed path', async () => { + getBuckets.mockReturnValueOnce({ + buckets: Array.from({ length: 60 }, () => ({ size: 1000, forEach: () => { } })), + totalSize: 60000, + }); + const precomputedCellIds = [10, 11, 12]; + getHeatmapCellOrder.mockReturnValueOnce(precomputedCellIds); + fetchWork.mockResolvedValueOnce(makeWorkerResponse({ cellIds: precomputedCellIds })); + + const store = mockStore(makeState(largeCellSets)); + await store.dispatch(loadHeatmapExpression(experimentId, ['GENE1'], options)); + + const [, body] = fetchWork.mock.calls[0]; + expect(Array.isArray(body.downsampleSettings.cellIds)).toBe(true); + expect(body.downsampleSettings.selectedCellSet).toBeUndefined(); + }); + + it('dispatches HEATMAP_EXPRESSION_ERROR on fetchWork failure', async () => { + getBuckets.mockReturnValueOnce({ + buckets: [new Set([0])], + totalSize: 1, + }); + fetchWork.mockRejectedValueOnce(new Error('worker failure')); + + const store = mockStore(makeState(largeCellSets)); + await store.dispatch(loadHeatmapExpression(experimentId, ['GENE1'], options)); + + const types = store.getActions().map((a) => a.type); + expect(types).toContain(HEATMAP_EXPRESSION_LOADING); + expect(types).toContain(HEATMAP_EXPRESSION_ERROR); + }); + + it('returns null without dispatching when all genes already loaded with unchanged settings', async () => { + getBuckets.mockReturnValueOnce({ + buckets: [new Set([0])], + totalSize: 1, + }); + + const genesInitial = getInitialState(); + // Pre-seed stored genes in the downsampled matrix + genesInitial.expression.downsampled.matrix.setGeneExpression( + ['GENE1'], + new SparseMatrix([[0]]), + { + rawMean: [0], rawStdev: [0], truncatedMin: [0], truncatedMax: [0], + }, + ); + genesInitial.expression.downsampled.lastFetchSettings = { + downsampleType: 'bucketed', + selectedCellSet: options.selectedCellSet, + groupedTracks: options.groupedTracks, + cellIdsKey: null, + }; + genesInitial.expression.downsampled.downsampleType = 'bucketed'; + + const store = mockStore(makeState(largeCellSets, { + downsampled: genesInitial.expression.downsampled, + })); + + const result = await store.dispatch( + loadHeatmapExpression(experimentId, ['GENE1'], options), + ); + + expect(result).toBeNull(); + expect(store.getActions()).toHaveLength(0); + expect(fetchWork).not.toHaveBeenCalled(); + }); + + it('uses stored cellIds in append request (settings unchanged, new genes)', async () => { + getBuckets.mockReturnValueOnce({ + buckets: [new Set([0])], + totalSize: 1, + }); + fetchWork.mockResolvedValueOnce(makeWorkerResponse({ orderedGeneNames: ['GENE2'] })); + + const storedCellIds = [0, 1, 2, 3]; + const genesInitial = getInitialState(); + // GENE1 already loaded + genesInitial.expression.downsampled.matrix.setGeneExpression( + ['GENE1'], + new SparseMatrix([[0]]), + { + rawMean: [0], rawStdev: [0], truncatedMin: [0], truncatedMax: [0], + }, + ); + genesInitial.expression.downsampled.lastFetchSettings = { + downsampleType: 'bucketed', + selectedCellSet: options.selectedCellSet, + groupedTracks: options.groupedTracks, + cellIdsKey: null, + }; + genesInitial.expression.downsampled.downsampleType = 'bucketed'; + genesInitial.expression.downsampled.cellIds = storedCellIds; + + const store = mockStore(makeState(largeCellSets, { + downsampled: genesInitial.expression.downsampled, + })); + + // Request GENE1 + GENE2 - GENE1 is already loaded, GENE2 is new + await store.dispatch( + loadHeatmapExpression(experimentId, ['GENE1', 'GENE2'], options), + ); + + // Should use stored cellIds in the request (append mode) + const [, body] = fetchWork.mock.calls[0]; + expect(body.downsampleSettings.cellIds).toEqual(storedCellIds); + expect(body.genes).toEqual(['GENE2']); // only the new gene + }); +}); diff --git a/src/__test__/redux/reducers/__snapshots__/genesReducer.test.js.snap b/src/__test__/redux/reducers/__snapshots__/genesReducer.test.js.snap index 1a7af1ae73..8ee0feded7 100644 --- a/src/__test__/redux/reducers/__snapshots__/genesReducer.test.js.snap +++ b/src/__test__/redux/reducers/__snapshots__/genesReducer.test.js.snap @@ -3,6 +3,36 @@ exports[`genesReducer Deselecting all genes updates to empty list 1`] = ` { "expression": { + "downsampled": { + "cellIds": [], + "downsampleType": null, + "error": false, + "lastFetchSettings": null, + "loading": false, + "matrix": ExpressionMatrix { + "geneIndexes": {}, + "rawGeneExpressions": { + "datatype": undefined, + "index": [], + "mathjs": "SparseMatrix", + "ptr": [ + 0, + ], + "size": [ + 0, + 0, + ], + "values": [], + }, + "setGeneExpression": [Function], + "stats": { + "rawMean": [], + "rawStdev": [], + "truncatedMax": [], + "truncatedMin": [], + }, + }, + }, "full": { "ETag": null, "error": false, @@ -51,6 +81,36 @@ exports[`genesReducer Deselecting all genes updates to empty list 1`] = ` exports[`genesReducer Deselecting non-selected genes is handled gracefully 1`] = ` { "expression": { + "downsampled": { + "cellIds": [], + "downsampleType": null, + "error": false, + "lastFetchSettings": null, + "loading": false, + "matrix": ExpressionMatrix { + "geneIndexes": {}, + "rawGeneExpressions": { + "datatype": undefined, + "index": [], + "mathjs": "SparseMatrix", + "ptr": [ + 0, + ], + "size": [ + 0, + 0, + ], + "values": [], + }, + "setGeneExpression": [Function], + "stats": { + "rawMean": [], + "rawStdev": [], + "truncatedMax": [], + "truncatedMin": [], + }, + }, + }, "full": { "ETag": null, "error": false, @@ -103,6 +163,36 @@ exports[`genesReducer Deselecting non-selected genes is handled gracefully 1`] = exports[`genesReducer Deselecting part of all genes updates list as set 1`] = ` { "expression": { + "downsampled": { + "cellIds": [], + "downsampleType": null, + "error": false, + "lastFetchSettings": null, + "loading": false, + "matrix": ExpressionMatrix { + "geneIndexes": {}, + "rawGeneExpressions": { + "datatype": undefined, + "index": [], + "mathjs": "SparseMatrix", + "ptr": [ + 0, + ], + "size": [ + 0, + 0, + ], + "values": [], + }, + "setGeneExpression": [Function], + "stats": { + "rawMean": [], + "rawStdev": [], + "truncatedMax": [], + "truncatedMin": [], + }, + }, + }, "full": { "ETag": null, "error": false, @@ -154,6 +244,36 @@ exports[`genesReducer Deselecting part of all genes updates list as set 1`] = ` exports[`genesReducer Error state handled appropriately 1`] = ` { "expression": { + "downsampled": { + "cellIds": [], + "downsampleType": null, + "error": false, + "lastFetchSettings": null, + "loading": false, + "matrix": ExpressionMatrix { + "geneIndexes": {}, + "rawGeneExpressions": { + "datatype": undefined, + "index": [], + "mathjs": "SparseMatrix", + "ptr": [ + 0, + ], + "size": [ + 0, + 0, + ], + "values": [], + }, + "setGeneExpression": [Function], + "stats": { + "rawMean": [], + "rawStdev": [], + "truncatedMax": [], + "truncatedMin": [], + }, + }, + }, "full": { "ETag": null, "error": false, @@ -208,6 +328,36 @@ exports[`genesReducer Error state handled appropriately 1`] = ` exports[`genesReducer Expression loaded state handled appropriately when other things are still loading 1`] = ` { "expression": { + "downsampled": { + "cellIds": [], + "downsampleType": null, + "error": false, + "lastFetchSettings": null, + "loading": false, + "matrix": ExpressionMatrix { + "geneIndexes": {}, + "rawGeneExpressions": { + "datatype": undefined, + "index": [], + "mathjs": "SparseMatrix", + "ptr": [ + 0, + ], + "size": [ + 0, + 0, + ], + "values": [], + }, + "setGeneExpression": [Function], + "stats": { + "rawMean": [], + "rawStdev": [], + "truncatedMax": [], + "truncatedMin": [], + }, + }, + }, "full": { "ETag": null, "error": false, @@ -304,6 +454,36 @@ exports[`genesReducer Expression loaded state handled appropriately when other t exports[`genesReducer Loaded paginated state handled appropriately 1`] = ` { "expression": { + "downsampled": { + "cellIds": [], + "downsampleType": null, + "error": false, + "lastFetchSettings": null, + "loading": false, + "matrix": ExpressionMatrix { + "geneIndexes": {}, + "rawGeneExpressions": { + "datatype": undefined, + "index": [], + "mathjs": "SparseMatrix", + "ptr": [ + 0, + ], + "size": [ + 0, + 0, + ], + "values": [], + }, + "setGeneExpression": [Function], + "stats": { + "rawMean": [], + "rawStdev": [], + "truncatedMax": [], + "truncatedMin": [], + }, + }, + }, "full": { "ETag": null, "error": false, @@ -373,6 +553,36 @@ exports[`genesReducer Loaded paginated state handled appropriately 1`] = ` exports[`genesReducer Loaded paginated state handled appropriately when other things are still loading 1`] = ` { "expression": { + "downsampled": { + "cellIds": [], + "downsampleType": null, + "error": false, + "lastFetchSettings": null, + "loading": false, + "matrix": ExpressionMatrix { + "geneIndexes": {}, + "rawGeneExpressions": { + "datatype": undefined, + "index": [], + "mathjs": "SparseMatrix", + "ptr": [ + 0, + ], + "size": [ + 0, + 0, + ], + "values": [], + }, + "setGeneExpression": [Function], + "stats": { + "rawMean": [], + "rawStdev": [], + "truncatedMax": [], + "truncatedMin": [], + }, + }, + }, "full": { "ETag": null, "error": false, @@ -445,6 +655,36 @@ exports[`genesReducer Loaded paginated state handled appropriately when other th exports[`genesReducer Loading on error state causes error to reset 1`] = ` { "expression": { + "downsampled": { + "cellIds": [], + "downsampleType": null, + "error": false, + "lastFetchSettings": null, + "loading": false, + "matrix": ExpressionMatrix { + "geneIndexes": {}, + "rawGeneExpressions": { + "datatype": undefined, + "index": [], + "mathjs": "SparseMatrix", + "ptr": [ + 0, + ], + "size": [ + 0, + 0, + ], + "values": [], + }, + "setGeneExpression": [Function], + "stats": { + "rawMean": [], + "rawStdev": [], + "truncatedMax": [], + "truncatedMin": [], + }, + }, + }, "full": { "ETag": null, "error": false, @@ -502,6 +742,36 @@ exports[`genesReducer Loading on error state causes error to reset 1`] = ` exports[`genesReducer Multiple components loading some of same expression triggers appropriate action 1`] = ` { "expression": { + "downsampled": { + "cellIds": [], + "downsampleType": null, + "error": false, + "lastFetchSettings": null, + "loading": false, + "matrix": ExpressionMatrix { + "geneIndexes": {}, + "rawGeneExpressions": { + "datatype": undefined, + "index": [], + "mathjs": "SparseMatrix", + "ptr": [ + 0, + ], + "size": [ + 0, + 0, + ], + "values": [], + }, + "setGeneExpression": [Function], + "stats": { + "rawMean": [], + "rawStdev": [], + "truncatedMax": [], + "truncatedMin": [], + }, + }, + }, "error": false, "full": { "ETag": null, @@ -566,6 +836,36 @@ exports[`genesReducer Multiple components loading some of same expression trigge exports[`genesReducer Multiple components loading some of same property triggers appropriate action 1`] = ` { "expression": { + "downsampled": { + "cellIds": [], + "downsampleType": null, + "error": false, + "lastFetchSettings": null, + "loading": false, + "matrix": ExpressionMatrix { + "geneIndexes": {}, + "rawGeneExpressions": { + "datatype": undefined, + "index": [], + "mathjs": "SparseMatrix", + "ptr": [ + 0, + ], + "size": [ + 0, + 0, + ], + "values": [], + }, + "setGeneExpression": [Function], + "stats": { + "rawMean": [], + "rawStdev": [], + "truncatedMax": [], + "truncatedMin": [], + }, + }, + }, "full": { "ETag": null, "error": false, @@ -629,6 +929,36 @@ exports[`genesReducer Multiple components loading some of same property triggers exports[`genesReducer Properties loading triggers appropriate changes 1`] = ` { "expression": { + "downsampled": { + "cellIds": [], + "downsampleType": null, + "error": false, + "lastFetchSettings": null, + "loading": false, + "matrix": ExpressionMatrix { + "geneIndexes": {}, + "rawGeneExpressions": { + "datatype": undefined, + "index": [], + "mathjs": "SparseMatrix", + "ptr": [ + 0, + ], + "size": [ + 0, + 0, + ], + "values": [], + }, + "setGeneExpression": [Function], + "stats": { + "rawMean": [], + "rawStdev": [], + "truncatedMax": [], + "truncatedMin": [], + }, + }, + }, "full": { "ETag": null, "error": false, @@ -686,6 +1016,36 @@ exports[`genesReducer Properties loading triggers appropriate changes 1`] = ` exports[`genesReducer Selected genes get added as a set to a non-empty list 1`] = ` { "expression": { + "downsampled": { + "cellIds": [], + "downsampleType": null, + "error": false, + "lastFetchSettings": null, + "loading": false, + "matrix": ExpressionMatrix { + "geneIndexes": {}, + "rawGeneExpressions": { + "datatype": undefined, + "index": [], + "mathjs": "SparseMatrix", + "ptr": [ + 0, + ], + "size": [ + 0, + 0, + ], + "values": [], + }, + "setGeneExpression": [Function], + "stats": { + "rawMean": [], + "rawStdev": [], + "truncatedMax": [], + "truncatedMin": [], + }, + }, + }, "full": { "ETag": null, "error": false, @@ -738,6 +1098,36 @@ exports[`genesReducer Selected genes get added as a set to a non-empty list 1`] exports[`genesReducer Selected genes get added on empty list 1`] = ` { "expression": { + "downsampled": { + "cellIds": [], + "downsampleType": null, + "error": false, + "lastFetchSettings": null, + "loading": false, + "matrix": ExpressionMatrix { + "geneIndexes": {}, + "rawGeneExpressions": { + "datatype": undefined, + "index": [], + "mathjs": "SparseMatrix", + "ptr": [ + 0, + ], + "size": [ + 0, + 0, + ], + "values": [], + }, + "setGeneExpression": [Function], + "stats": { + "rawMean": [], + "rawStdev": [], + "truncatedMax": [], + "truncatedMin": [], + }, + }, + }, "full": { "ETag": null, "error": false, @@ -790,6 +1180,36 @@ exports[`genesReducer Selected genes get added on empty list 1`] = ` exports[`genesReducer Sets error state on expression error action 1`] = ` { "expression": { + "downsampled": { + "cellIds": [], + "downsampleType": null, + "error": false, + "lastFetchSettings": null, + "loading": false, + "matrix": ExpressionMatrix { + "geneIndexes": {}, + "rawGeneExpressions": { + "datatype": undefined, + "index": [], + "mathjs": "SparseMatrix", + "ptr": [ + 0, + ], + "size": [ + 0, + 0, + ], + "values": [], + }, + "setGeneExpression": [Function], + "stats": { + "rawMean": [], + "rawStdev": [], + "truncatedMax": [], + "truncatedMin": [], + }, + }, + }, "full": { "ETag": null, "error": "asd", @@ -844,6 +1264,36 @@ exports[`genesReducer Sets error state on expression error action 1`] = ` exports[`genesReducer Sets loaded state on expression loading action 1`] = ` { "expression": { + "downsampled": { + "cellIds": [], + "downsampleType": null, + "error": false, + "lastFetchSettings": null, + "loading": false, + "matrix": ExpressionMatrix { + "geneIndexes": {}, + "rawGeneExpressions": { + "datatype": undefined, + "index": [], + "mathjs": "SparseMatrix", + "ptr": [ + 0, + ], + "size": [ + 0, + 0, + ], + "values": [], + }, + "setGeneExpression": [Function], + "stats": { + "rawMean": [], + "rawStdev": [], + "truncatedMax": [], + "truncatedMin": [], + }, + }, + }, "full": { "ETag": null, "error": false, @@ -926,6 +1376,36 @@ exports[`genesReducer Sets loaded state on expression loading action 1`] = ` exports[`genesReducer Sets loading state on expression loading action 1`] = ` { "expression": { + "downsampled": { + "cellIds": [], + "downsampleType": null, + "error": false, + "lastFetchSettings": null, + "loading": false, + "matrix": ExpressionMatrix { + "geneIndexes": {}, + "rawGeneExpressions": { + "datatype": undefined, + "index": [], + "mathjs": "SparseMatrix", + "ptr": [ + 0, + ], + "size": [ + 0, + 0, + ], + "values": [], + }, + "setGeneExpression": [Function], + "stats": { + "rawMean": [], + "rawStdev": [], + "truncatedMax": [], + "truncatedMin": [], + }, + }, + }, "error": false, "full": { "ETag": null, diff --git a/src/__test__/redux/reducers/genes/__snapshots__/markerGenesLoaded.test.js.snap b/src/__test__/redux/reducers/genes/__snapshots__/markerGenesLoaded.test.js.snap index 1c12cfea0e..d3234efbbc 100644 --- a/src/__test__/redux/reducers/genes/__snapshots__/markerGenesLoaded.test.js.snap +++ b/src/__test__/redux/reducers/genes/__snapshots__/markerGenesLoaded.test.js.snap @@ -3,6 +3,36 @@ exports[`markerGenesLoaded returns correct state 1`] = ` { "expression": { + "downsampled": { + "cellIds": [], + "downsampleType": null, + "error": false, + "lastFetchSettings": null, + "loading": false, + "matrix": ExpressionMatrix { + "geneIndexes": {}, + "rawGeneExpressions": { + "datatype": undefined, + "index": [], + "mathjs": "SparseMatrix", + "ptr": [ + 0, + ], + "size": [ + 0, + 0, + ], + "values": [], + }, + "setGeneExpression": [Function], + "stats": { + "rawMean": [], + "rawStdev": [], + "truncatedMax": [], + "truncatedMin": [], + }, + }, + }, "full": { "ETag": null, "error": false, diff --git a/src/__test__/redux/reducers/genes/heatmapExpressionError.test.js b/src/__test__/redux/reducers/genes/heatmapExpressionError.test.js new file mode 100644 index 0000000000..0f98acdbff --- /dev/null +++ b/src/__test__/redux/reducers/genes/heatmapExpressionError.test.js @@ -0,0 +1,43 @@ +import heatmapExpressionError from 'redux/reducers/genes/heatmapExpressionError'; + +describe('heatmapExpressionError', () => { + const baseState = { + expression: { + full: { matrix: {}, loading: [], error: false }, + downsampled: { + loading: true, + error: false, + matrix: {}, + cellIds: [1, 2, 3], + downsampleType: 'bucketed', + lastFetchSettings: { selectedCellSet: 'louvain', groupedTracks: ['sample'] }, + }, + }, + }; + + const testError = new Error('Work request failed'); + + it('sets downsampled.error to the error from action payload', () => { + const newState = heatmapExpressionError(baseState, { payload: { error: testError } }); + expect(newState.expression.downsampled.error).toBe(testError); + }); + + it('sets downsampled.loading to false', () => { + const newState = heatmapExpressionError(baseState, { payload: { error: testError } }); + expect(newState.expression.downsampled.loading).toBe(false); + }); + + it('preserves existing downsampled data (cellIds, downsampleType, lastFetchSettings)', () => { + const newState = heatmapExpressionError(baseState, { payload: { error: testError } }); + expect(newState.expression.downsampled.cellIds).toEqual([1, 2, 3]); + expect(newState.expression.downsampled.downsampleType).toBe('bucketed'); + expect(newState.expression.downsampled.lastFetchSettings).toEqual({ + selectedCellSet: 'louvain', groupedTracks: ['sample'], + }); + }); + + it('preserves full expression state', () => { + const newState = heatmapExpressionError(baseState, { payload: { error: testError } }); + expect(newState.expression.full).toEqual(baseState.expression.full); + }); +}); diff --git a/src/__test__/redux/reducers/genes/heatmapExpressionLoaded.test.js b/src/__test__/redux/reducers/genes/heatmapExpressionLoaded.test.js new file mode 100644 index 0000000000..4892f295cc --- /dev/null +++ b/src/__test__/redux/reducers/genes/heatmapExpressionLoaded.test.js @@ -0,0 +1,141 @@ +import { SparseMatrix } from 'mathjs'; +import ExpressionMatrix from 'utils/ExpressionMatrix/ExpressionMatrix'; +import heatmapExpressionLoaded from 'redux/reducers/genes/heatmapExpressionLoaded'; + +const makeMatrix = () => new ExpressionMatrix(); + +// Minimal valid gene data (1 gene, 3 cells) for setGeneExpression / pushGeneExpression +const makeGeneData = (geneName = 'GENE1') => ({ + orderedGeneNames: [geneName], + rawExpression: new SparseMatrix([[0], [1], [2]]), + stats: { + rawMean: [1], + rawStdev: [0.5], + truncatedMin: [0], + truncatedMax: [2], + }, +}); + +const makeAction = (overrides = {}) => ({ + payload: { + newGenes: null, + cellIds: [0, 1, 2], + downsampleType: 'bucketed', + lastFetchSettings: { + downsampleType: 'bucketed', + selectedCellSet: 'louvain', + groupedTracks: ['sample'], + cellIdsKey: null, + }, + isAppend: false, + genesAlreadyLoaded: [], + ...overrides, + }, +}); + +describe('heatmapExpressionLoaded', () => { + let baseState; + + beforeEach(() => { + baseState = { + expression: { + full: { matrix: makeMatrix(), loading: [], error: false }, + downsampled: { + loading: true, + error: false, + matrix: makeMatrix(), + cellIds: null, + downsampleType: null, + lastFetchSettings: null, + }, + }, + }; + }); + + it('sets downsampled.loading to false', () => { + const newState = heatmapExpressionLoaded(baseState, makeAction()); + expect(newState.expression.downsampled.loading).toBe(false); + }); + + it('clears downsampled.error', () => { + baseState.expression.downsampled.error = new Error('previous error'); + const newState = heatmapExpressionLoaded(baseState, makeAction()); + expect(newState.expression.downsampled.error).toBe(false); + }); + + it('stores cellIds when not an append and newGenes is provided', () => { + const action = makeAction({ + newGenes: makeGeneData('GENE1'), + cellIds: [10, 20, 30], + isAppend: false, + }); + const newState = heatmapExpressionLoaded(baseState, action); + expect(newState.expression.downsampled.cellIds).toEqual([10, 20, 30]); + }); + + it('stores downsampleType when not an append and newGenes is provided', () => { + const action = makeAction({ + newGenes: makeGeneData('GENE1'), + downsampleType: 'precomputed', + isAppend: false, + }); + const newState = heatmapExpressionLoaded(baseState, action); + expect(newState.expression.downsampled.downsampleType).toBe('precomputed'); + }); + + it('stores lastFetchSettings when not an append and newGenes is provided', () => { + const settings = { + downsampleType: 'bucketed', + selectedCellSet: 'louvain', + groupedTracks: ['sample'], + cellIdsKey: null, + }; + const action = makeAction({ + newGenes: makeGeneData('GENE1'), + lastFetchSettings: settings, + isAppend: false, + }); + const newState = heatmapExpressionLoaded(baseState, action); + expect(newState.expression.downsampled.lastFetchSettings).toEqual(settings); + }); + + it('does not overwrite cellIds/downsampleType/lastFetchSettings in append mode', () => { + const existingCellIds = [1, 2, 3]; + baseState.expression.downsampled.cellIds = existingCellIds; + baseState.expression.downsampled.downsampleType = 'bucketed'; + baseState.expression.downsampled.lastFetchSettings = { selectedCellSet: 'louvain' }; + + const action = makeAction({ + newGenes: makeGeneData('GENE2'), + cellIds: [99], // should NOT be used for append + downsampleType: 'precomputed', + lastFetchSettings: { selectedCellSet: 'sample' }, + isAppend: true, + }); + const newState = heatmapExpressionLoaded(baseState, action); + expect(newState.expression.downsampled.cellIds).toEqual(existingCellIds); + expect(newState.expression.downsampled.downsampleType).toBe('bucketed'); + expect(newState.expression.downsampled.lastFetchSettings).toEqual({ selectedCellSet: 'louvain' }); + }); + + it('preserves full expression state', () => { + const action = makeAction(); + const newState = heatmapExpressionLoaded(baseState, action); + expect(newState.expression.full).toBe(baseState.expression.full); + }); + + it('does not overwrite metadata when newGenes is null', () => { + baseState.expression.downsampled.cellIds = [5, 6, 7]; + baseState.expression.downsampled.downsampleType = 'precomputed'; + + const action = makeAction({ + newGenes: null, + cellIds: [0, 1, 2], + isAppend: false, + }); + const newState = heatmapExpressionLoaded(baseState, action); + // When newGenes is null, the conditional `!isAppend && newGenes && {}` doesn't spread + expect(newState.expression.downsampled.cellIds).toEqual([5, 6, 7]); + expect(newState.expression.downsampled.downsampleType).toBe('precomputed'); + }); +}); diff --git a/src/__test__/redux/reducers/genes/heatmapExpressionLoading.test.js b/src/__test__/redux/reducers/genes/heatmapExpressionLoading.test.js new file mode 100644 index 0000000000..f8980308df --- /dev/null +++ b/src/__test__/redux/reducers/genes/heatmapExpressionLoading.test.js @@ -0,0 +1,60 @@ +import heatmapExpressionLoading from 'redux/reducers/genes/heatmapExpressionLoading'; + +describe('heatmapExpressionLoading', () => { + const baseState = { + expression: { + full: { matrix: {}, loading: [], error: false }, + downsampled: { + loading: false, + error: false, + matrix: {}, + cellIds: null, + downsampleType: null, + lastFetchSettings: null, + }, + }, + }; + + it('sets downsampled.loading to true', () => { + const newState = heatmapExpressionLoading(baseState); + expect(newState.expression.downsampled.loading).toBe(true); + }); + + it('clears downsampled.error', () => { + const stateWithError = { + ...baseState, + expression: { + ...baseState.expression, + downsampled: { ...baseState.expression.downsampled, error: new Error('previous') }, + }, + }; + const newState = heatmapExpressionLoading(stateWithError); + expect(newState.expression.downsampled.error).toBe(false); + }); + + it('preserves rest of downsampled state', () => { + const stateWithData = { + ...baseState, + expression: { + ...baseState.expression, + downsampled: { + ...baseState.expression.downsampled, + cellIds: [1, 2, 3], + downsampleType: 'bucketed', + lastFetchSettings: { selectedCellSet: 'louvain', groupedTracks: ['sample'] }, + }, + }, + }; + const newState = heatmapExpressionLoading(stateWithData); + expect(newState.expression.downsampled.cellIds).toEqual([1, 2, 3]); + expect(newState.expression.downsampled.downsampleType).toBe('bucketed'); + expect(newState.expression.downsampled.lastFetchSettings).toEqual({ + selectedCellSet: 'louvain', groupedTracks: ['sample'], + }); + }); + + it('preserves full expression state', () => { + const newState = heatmapExpressionLoading(baseState); + expect(newState.expression.full).toEqual(baseState.expression.full); + }); +}); diff --git a/src/__test__/utils/plotSpecs/__snapshots__/generateEmbeddingCategoricalSpec.test.js.snap b/src/__test__/utils/plotSpecs/__snapshots__/generateEmbeddingCategoricalSpec.test.js.snap index 9d15fd3761..46658dbfc0 100644 --- a/src/__test__/utils/plotSpecs/__snapshots__/generateEmbeddingCategoricalSpec.test.js.snap +++ b/src/__test__/utils/plotSpecs/__snapshots__/generateEmbeddingCategoricalSpec.test.js.snap @@ -1099,128 +1099,7 @@ exports[`generateData generates correct data with 1 cluster 1`] = ` "name": "Cluster 5", }, ], - "plotData": [ - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - ], + "plotData": [], } `; @@ -1299,10 +1178,6 @@ exports[`generateData generates correct data with all louvain clusters 1`] = ` }, ], "plotData": [ - { - "x": 1, - "y": 2, - }, { "cellId": 1, "cellSetKey": "louvain-0", @@ -1399,18 +1274,6 @@ exports[`generateData generates correct data with all louvain clusters 1`] = ` "x": 1, "y": 2, }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, - { - "x": 1, - "y": 2, - }, { "cellId": 16, "cellSetKey": "louvain-0", diff --git a/src/__test__/utils/work/getHeatmapCellOrder.test.js b/src/__test__/utils/work/getHeatmapCellOrder.test.js index 7c4e6c1815..bb55bc34c1 100644 --- a/src/__test__/utils/work/getHeatmapCellOrder.test.js +++ b/src/__test__/utils/work/getHeatmapCellOrder.test.js @@ -1,4 +1,8 @@ -import getHeatmapCellOrder from 'utils/work/getHeatmapCellOrder'; +import getHeatmapCellOrder, { + getBuckets, + computeBucketedDisplayCellIds, + computeHiddenCellSets, +} from 'utils/work/getHeatmapCellOrder'; describe('getHeatmapCellOrder', () => { // Mock cell sets structure @@ -349,3 +353,370 @@ describe('getHeatmapCellOrder', () => { expect(new Set(result)).toEqual(new Set([5, 6, 10, 11, 14])); }); }); + +describe('getBuckets', () => { + const mockCellSets = { + hierarchy: [ + { + key: 'louvain', + children: [ + { key: 'louvain-0' }, + { key: 'louvain-1' }, + { key: 'louvain-2' }, + ], + }, + { + key: 'sample', + children: [ + { key: 'sample-1' }, + { key: 'sample-2' }, + ], + }, + { + key: 'patient', + children: [ + { key: 'patient-A' }, + { key: 'patient-B' }, + ], + }, + ], + properties: { + 'louvain-0': { cellIds: new Set([0, 1, 2, 3, 4]) }, + 'louvain-1': { cellIds: new Set([5, 6, 7, 8, 9]) }, + 'louvain-2': { cellIds: new Set([10, 11, 12, 13, 14]) }, + 'sample-1': { cellIds: new Set([0, 1, 4, 5, 6, 10, 11, 14]) }, + 'sample-2': { cellIds: new Set([2, 3, 7, 8, 9, 12, 13]) }, + 'patient-A': { cellIds: new Set([0, 2, 4, 5, 7, 10, 12]) }, + 'patient-B': { cellIds: new Set([1, 3, 6, 8, 9, 11, 13, 14]) }, + }, + }; + + it('returns empty buckets and zero totalSize for null cellSets', () => { + expect(getBuckets('louvain', ['sample'], [], null)).toEqual({ buckets: [], totalSize: 0 }); + expect(getBuckets('louvain', ['sample'], [], undefined)).toEqual({ buckets: [], totalSize: 0 }); + }); + + it('returns empty buckets and zero totalSize for missing groupedTracks', () => { + expect(getBuckets('louvain', null, [], mockCellSets)).toEqual({ buckets: [], totalSize: 0 }); + }); + + it('returns empty buckets when groupedTracks is empty', () => { + const { buckets, totalSize } = getBuckets('louvain', [], [], mockCellSets); + expect(buckets).toEqual([]); + expect(totalSize).toBe(0); + }); + + it('returns empty buckets for invalid selectedCellSet', () => { + const { buckets, totalSize } = getBuckets('invalid-key', ['sample'], [], mockCellSets); + expect(buckets).toEqual([]); + expect(totalSize).toBe(0); + }); + + it('returns correct bucket count for single grouped track', () => { + // One track 'sample' → 2 children → 2 buckets (sample-1 and sample-2) + const { buckets, totalSize } = getBuckets('louvain', ['sample'], [], mockCellSets); + expect(buckets).toHaveLength(2); + expect(totalSize).toBe(15); + }); + + it('returns cartesian product buckets for two grouped tracks', () => { + // Two tracks → at most 2×2=4 buckets (louvain×sample cartesian product) + const { buckets, totalSize } = getBuckets('louvain', ['sample', 'patient'], [], mockCellSets); + expect(buckets).toHaveLength(4); + expect(totalSize).toBe(15); + }); + + it('totalSize equals the sum of all bucket sizes', () => { + const { buckets, totalSize } = getBuckets('louvain', ['sample', 'patient'], [], mockCellSets); + const computedTotal = buckets.reduce((sum, bucket) => sum + bucket.size, 0); + expect(computedTotal).toBe(totalSize); + }); + + it('each bucket is a Set instance', () => { + const { buckets } = getBuckets('louvain', ['sample'], [], mockCellSets); + buckets.forEach((bucket) => expect(bucket).toBeInstanceOf(Set)); + }); + + it('buckets collectively contain all enabled cells when no hidden sets', () => { + const { buckets } = getBuckets('louvain', ['sample'], [], mockCellSets); + const allCells = new Set(); + buckets.forEach((bucket) => bucket.forEach((id) => allCells.add(id))); + expect(allCells).toEqual(new Set([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14])); + }); + + it('no cell appears in more than one bucket', () => { + const { buckets } = getBuckets('louvain', ['sample', 'patient'], [], mockCellSets); + const seen = new Set(); + buckets.forEach((bucket) => { + bucket.forEach((id) => { + expect(seen.has(id)).toBe(false); + seen.add(id); + }); + }); + }); + + it('hidden cell sets are excluded from buckets', () => { + // Hide louvain-0 ([0,1,2,3,4]) → only 10 cells remain + const { buckets, totalSize } = getBuckets('louvain', ['sample'], ['louvain-0'], mockCellSets); + expect(totalSize).toBe(10); + const allCells = new Set(); + buckets.forEach((bucket) => bucket.forEach((id) => allCells.add(id))); + [0, 1, 2, 3, 4].forEach((id) => expect(allCells.has(id)).toBe(false)); + }); + + it('returns empty buckets when all cells are hidden', () => { + const { buckets, totalSize } = getBuckets( + 'louvain', ['sample'], ['louvain-0', 'louvain-1', 'louvain-2'], mockCellSets, + ); + expect(buckets).toEqual([]); + expect(totalSize).toBe(0); + }); + + it('accepts hiddenCellSets as a Set', () => { + const { totalSize: sizeFromSet } = getBuckets( + 'louvain', ['sample'], new Set(['louvain-0']), mockCellSets, + ); + const { totalSize: sizeFromArray } = getBuckets( + 'louvain', ['sample'], ['louvain-0'], mockCellSets, + ); + expect(sizeFromSet).toBe(sizeFromArray); + }); +}); + +describe('computeBucketedDisplayCellIds', () => { + const mockCellSets = { + hierarchy: [ + { + key: 'louvain', + children: [ + { key: 'louvain-0' }, + { key: 'louvain-1' }, + { key: 'louvain-2' }, + ], + }, + { + key: 'sample', + children: [ + { key: 'sample-1' }, + { key: 'sample-2' }, + ], + }, + { + key: 'patient', + children: [ + { key: 'patient-A' }, + { key: 'patient-B' }, + ], + }, + ], + properties: { + 'louvain-0': { cellIds: new Set([0, 1, 2, 3, 4]) }, + 'louvain-1': { cellIds: new Set([5, 6, 7, 8, 9]) }, + 'louvain-2': { cellIds: new Set([10, 11, 12, 13, 14]) }, + 'sample-1': { cellIds: new Set([0, 1, 4, 5, 6, 10, 11, 14]) }, + 'sample-2': { cellIds: new Set([2, 3, 7, 8, 9, 12, 13]) }, + 'patient-A': { cellIds: new Set([0, 2, 4, 5, 7, 10, 12]) }, + 'patient-B': { cellIds: new Set([1, 3, 6, 8, 9, 11, 13, 14]) }, + }, + }; + + const allCellIds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]; + + it('returns empty array for empty workerCellIds', () => { + expect( + computeBucketedDisplayCellIds('louvain', ['sample'], [], mockCellSets, []), + ).toEqual([]); + }); + + it('returns empty array for null workerCellIds', () => { + expect( + computeBucketedDisplayCellIds('louvain', ['sample'], [], mockCellSets, null), + ).toEqual([]); + }); + + it('returns empty array for null cellSets', () => { + expect( + computeBucketedDisplayCellIds('louvain', ['sample'], [], null, allCellIds), + ).toEqual([]); + }); + + it('all returned cell IDs are present in workerCellIds', () => { + const workerCellIds = [0, 1, 5, 6, 10, 11]; // subset + const result = computeBucketedDisplayCellIds( + 'louvain', ['sample'], [], mockCellSets, workerCellIds, + ); + const workerSet = new Set(workerCellIds); + result.forEach((id) => expect(workerSet.has(id)).toBe(true)); + }); + + it('respects maxCells limit', () => { + const result = computeBucketedDisplayCellIds( + 'louvain', ['sample'], [], mockCellSets, allCellIds, 5, + ); + expect(result.length).toBeLessThanOrEqual(5); + }); + + it('hidden cells are excluded from results', () => { + // Hide louvain-0 ([0,1,2,3,4]) + const result = computeBucketedDisplayCellIds( + 'louvain', ['sample'], ['louvain-0'], mockCellSets, allCellIds, + ); + const hiddenIds = new Set([0, 1, 2, 3, 4]); + result.forEach((id) => expect(hiddenIds.has(id)).toBe(false)); + }); + + it('returns no duplicate cell IDs', () => { + const result = computeBucketedDisplayCellIds( + 'louvain', ['sample'], [], mockCellSets, allCellIds, + ); + expect(new Set(result).size).toBe(result.length); + }); + + it('is deterministic - same inputs produce same output', () => { + const result1 = computeBucketedDisplayCellIds( + 'louvain', ['sample'], [], mockCellSets, allCellIds, 8, + ); + const result2 = computeBucketedDisplayCellIds( + 'louvain', ['sample'], [], mockCellSets, allCellIds, 8, + ); + expect(result1).toEqual(result2); + }); + + it('only samples from workerCellIds when it is a subset of all cells', () => { + // Worker only returned cells from sample-1 + const workerSubset = [0, 1, 4, 5, 6, 10, 11, 14]; + const result = computeBucketedDisplayCellIds( + 'louvain', ['sample'], [], mockCellSets, workerSubset, + ); + const workerSet = new Set(workerSubset); + result.forEach((id) => expect(workerSet.has(id)).toBe(true)); + }); + + it('returns all eligible cells when under maxCells', () => { + // With 6 worker cells and default maxCells=5000, should return all 6 + const workerCellIds = [0, 5, 10, 2, 7, 12]; + const result = computeBucketedDisplayCellIds( + 'louvain', ['sample'], [], mockCellSets, workerCellIds, + ); + expect(result.length).toBe(6); + expect(new Set(result)).toEqual(new Set(workerCellIds)); + }); + + it('represents multiple buckets in the output', () => { + // All worker cells available, downsampled to 8 + // sample-1 has 8 cells, sample-2 has 7 cells - both buckets should be represented + const result = computeBucketedDisplayCellIds( + 'louvain', ['sample'], [], mockCellSets, allCellIds, 8, + ); + const sample1Ids = new Set([0, 1, 4, 5, 6, 10, 11, 14]); + const sample2Ids = new Set([2, 3, 7, 8, 9, 12, 13]); + const fromSample1 = result.filter((id) => sample1Ids.has(id)); + const fromSample2 = result.filter((id) => sample2Ids.has(id)); + // Both buckets should be represented + expect(fromSample1.length).toBeGreaterThan(0); + expect(fromSample2.length).toBeGreaterThan(0); + }); +}); + +describe('computeHiddenCellSets', () => { + const mockCellSets = { + hierarchy: [ + { + key: 'louvain', + children: [ + { key: 'louvain-0' }, + { key: 'louvain-1' }, + { key: 'louvain-2' }, + ], + }, + { + key: 'sample', + children: [ + { key: 'sample-1' }, + { key: 'sample-2' }, + ], + }, + ], + properties: {}, + hidden: new Set(), + }; + + it('returns empty array when selectedPoints is All and no hidden sets', () => { + const result = computeHiddenCellSets('All', mockCellSets); + expect(result).toEqual([]); + }); + + it('returns existing hidden sets when selectedPoints is All', () => { + const cellSetsWithHidden = { + ...mockCellSets, + hidden: new Set(['louvain-0', 'sample-1']), + }; + const result = computeHiddenCellSets('All', cellSetsWithHidden); + expect(result).toEqual(expect.arrayContaining(['louvain-0', 'sample-1'])); + expect(result).toHaveLength(2); + }); + + it('hides sibling cell sets when a specific selectedPoints is chosen', () => { + // selectedPoints='sample/sample-1' → sample-2 should be hidden + const result = computeHiddenCellSets('sample/sample-1', mockCellSets); + expect(result).toContain('sample-2'); + expect(result).not.toContain('sample-1'); + }); + + it('keeps selected cell set out of hidden list even if it was hidden before', () => { + // If sample-1 was already in hidden, selectedPoints='sample/sample-1' should remove it + const cellSetsWithSample1Hidden = { + ...mockCellSets, + hidden: new Set(['sample-1']), + }; + const result = computeHiddenCellSets('sample/sample-1', cellSetsWithSample1Hidden); + expect(result).not.toContain('sample-1'); + }); + + it('merges cellSets.hidden with selectedPoints-derived hidden sets', () => { + // louvain-0 is already hidden; selectedPoints hides sample-2 too + const cellSetsWithHidden = { + ...mockCellSets, + hidden: new Set(['louvain-0']), + }; + const result = computeHiddenCellSets('sample/sample-1', cellSetsWithHidden); + expect(result).toContain('louvain-0'); + expect(result).toContain('sample-2'); + expect(result).not.toContain('sample-1'); + }); + + it('does not add duplicate entries', () => { + // sample-2 is already in hidden; selectedPoints='sample/sample-1' also hides sample-2 + const cellSetsWithHidden = { + ...mockCellSets, + hidden: new Set(['sample-2']), + }; + const result = computeHiddenCellSets('sample/sample-1', cellSetsWithHidden); + expect(result.filter((k) => k === 'sample-2')).toHaveLength(1); + }); + + it('returns only cellSets.hidden for null selectedPoints', () => { + const cellSetsWithHidden = { + ...mockCellSets, + hidden: new Set(['louvain-0']), + }; + const result = computeHiddenCellSets(null, cellSetsWithHidden); + expect(result).toEqual(['louvain-0']); + }); + + it('returns only cellSets.hidden for undefined selectedPoints', () => { + const cellSetsWithHidden = { + ...mockCellSets, + hidden: new Set(['louvain-1']), + }; + const result = computeHiddenCellSets(undefined, cellSetsWithHidden); + expect(result).toEqual(['louvain-1']); + }); + + it('handles selectedPoints with category key not in hierarchy gracefully', () => { + // 'nonexistent/cell-0' → categoryKey 'nonexistent' not in hierarchy + const result = computeHiddenCellSets('nonexistent/cell-0', mockCellSets); + // Should not throw; returns hidden list unchanged + expect(result).toEqual([]); + }); +}); diff --git a/src/components/data-exploration/embedding/Embedding.jsx b/src/components/data-exploration/embedding/Embedding.jsx index 9b9790050e..13ba4def83 100644 --- a/src/components/data-exploration/embedding/Embedding.jsx +++ b/src/components/data-exploration/embedding/Embedding.jsx @@ -7,6 +7,7 @@ import { useSelector, useDispatch } from 'react-redux'; import PropTypes from 'prop-types'; import dynamic from 'next/dynamic'; import * as vega from 'vega'; +import { WebMercatorViewport } from '@deck.gl/core'; import { ScatterplotLayer } from '@deck.gl/layers'; import { EditableGeoJsonLayer } from '@nebula.gl/layers'; import { DrawPolygonByDraggingMode } from '@nebula.gl/edit-modes'; @@ -227,6 +228,19 @@ const Embedding = (props) => { setConvertedCellsData(convertCellsData(data, cellSetHidden, cellSetProperties)); }, [data, cellSetHidden, cellSetProperties]); + // Transform cell data for deck.gl + const deckglData = useMemo( + () => transformCellData(convertedCellsData, cellColors), + [convertedCellsData, cellColors], + ); + + // Map cellId → data-space position for fast lookup during crosshair projection + const cellIdToPositionMap = useMemo(() => { + const map = new Map(); + deckglData.forEach((d) => { map.set(String(d.cellId), d.position); }); + return map; + }, [deckglData]); + // Build quadtree from cell data for efficient lasso selection useEffect(() => { @@ -266,10 +280,36 @@ const Embedding = (props) => { expression: expressionToDispatch, geneName, }); + + // Project the selected cell's data-space coordinates to screen coordinates so the + // crosshair updates correctly when the selection originates from the heatmap. + const position = cellIdToPositionMap.get(String(selectedCell)); + if (position && viewState) { + try { + const viewport = new WebMercatorViewport({ + width, + height, + longitude: viewState.longitude, + latitude: viewState.latitude, + zoom: viewState.zoom, + pitch: viewState.pitch || 0, + bearing: viewState.bearing || 0, + }); + const [screenX, screenY] = viewport.project(position); + cellCoordinatesRef.current = { + x: screenX, + y: screenY, + width, + height, + }; + } catch (_e) { + // Projection can fail for cells outside the current viewport; keep existing coords + } + } } else { setCellInfoTooltip(null); } - }, [selectedCell]); + }, [selectedCell, cellIdToPositionMap]); const setCellHighlight = useCallback((cell) => { // Keep last shown tooltip @@ -310,12 +350,6 @@ const Embedding = (props) => { } }, [setCellHighlight, clearCellHighlight, width, height]); - // Transform cell data for deck.gl - const deckglData = useMemo( - () => transformCellData(convertedCellsData, cellColors), - [convertedCellsData, cellColors], - ); - // Auto-fit view when embedding data loads (not when colors change) useEffect(() => { if (deckglData && deckglData.length > 0 && !viewState) { @@ -388,7 +422,7 @@ const Embedding = (props) => { radiusScale: Math.pow(2, viewState.zoom - 10), radiusMinPixels: radiusMinPixels, radiusUnits: 'common', - radiusMaxPixels: isLargeDataset ? 2 : 6, + radiusMaxPixels: isLargeDataset ? 4 : 6, updateTriggers: { radiusScale: [viewState.zoom], }, @@ -529,59 +563,59 @@ const Embedding = (props) => { } }} > - {data && deckglData.length > 0 ? ( - <> - 0 ? ( + <> + +
+ {viewState && ( + setViewState(e.viewState)} + controller={activeTool === 'polygon' ? { scrollZoom: true, dragPan: false, dragRotate: false, touchZoom: true, touchRotate: false } : true} + layers={layers} + onHover={activeTool !== 'polygon' ? handleDeckGLHover : null} + getCursor={() => activeTool === 'polygon' ? 'crosshair' : 'default'} + style={{ width: '100%', height: '100%' }} + /> + )} +
+ + ) : ( + <> + )} + {renderExpressionView()} + { + createClusterPopover + ? ( + setCreateClusterPopover(false)} /> -
- {viewState && ( - setViewState(e.viewState)} - controller={activeTool === 'polygon' ? { scrollZoom: true, dragPan: false, dragRotate: false, touchZoom: true, touchRotate: false } : true} - layers={layers} - onHover={activeTool !== 'polygon' ? handleDeckGLHover : null} - getCursor={() => activeTool === 'polygon' ? 'crosshair' : 'default'} - style={{ width: '100%', height: '100%' }} + ) : ( + (cellInfoVisible && cellInfoTooltip && activeTool !== 'polygon') ? ( +
+ - )} -
- - ) : ( - <> - )} - {renderExpressionView()} - { - createClusterPopover - ? ( - setCreateClusterPopover(false)} - /> - ) : ( - (cellInfoVisible && cellInfoTooltip && activeTool !== 'polygon') ? ( -
- - -
- ) : <> - ) - } + +
+ ) : <> + ) + } {showLoader && (
{ newGenes = displayedGenes.filter((gene) => !selectedGenes.includes(gene)); } - // Update config with new gene list + // Update config with new gene list — the component's own effect handles expression loading dispatch(updatePlotConfig(componentType, { selectedGenes: newGenes })); - - dispatch(loadGeneExpression(experimentId, newGenes, componentType)); }; const menu = ( diff --git a/src/components/data-exploration/heatmap/HeatmapPlot.jsx b/src/components/data-exploration/heatmap/HeatmapPlot.jsx index fb97b59acd..44a6e53086 100644 --- a/src/components/data-exploration/heatmap/HeatmapPlot.jsx +++ b/src/components/data-exploration/heatmap/HeatmapPlot.jsx @@ -12,8 +12,10 @@ import { } from 'redux/selectors'; import { - loadGeneExpression, loadMarkerGenes, + loadHeatmapExpression, loadMarkerGenes, } from 'redux/actions/genes'; +import { LARGE_DATASET_THRESHOLD } from 'redux/actions/genes/loadHeatmapExpression'; +import { computeBucketedDisplayCellIds } from 'utils/work/getHeatmapCellOrder'; import { loadComponentConfig } from 'redux/actions/componentConfig'; import { updateCellInfo } from 'redux/actions/cellInfo'; @@ -27,9 +29,9 @@ import getContainingCellSetsProperties from 'utils/cellSets/getContainingCellSet import useConditionalEffect from 'utils/customHooks/useConditionalEffect'; import generateVitessceData from 'components/plots/helpers/heatmap/vitessce/generateVitessceData'; import { loadCellSets } from 'redux/actions/cellSets'; -import calculateNMarkerGenes from 'utils/calculateNMarkerGenes'; const COMPONENT_TYPE = 'interactiveHeatmap'; +const NUM_MARKER_GENES = 5; const Heatmap = dynamic( () => import('../DynamicVitessceWrappers').then((mod) => mod.Heatmap), @@ -47,9 +49,9 @@ const HeatmapPlot = (props) => { const dispatch = useDispatch(); - const debouncedLoadGeneExpression = useMemo(() => { + const debouncedLoadHeatmapExpression = useMemo(() => { const debounced = _.debounce((...params) => { - dispatch(loadGeneExpression(...params)); + dispatch(loadHeatmapExpression(...params)); }, 1000); return debounced; }, [dispatch]); @@ -57,11 +59,19 @@ const HeatmapPlot = (props) => { // Cancel pending debounced calls on unmount to prevent unexpected dispatches useEffect(() => { return () => { - debouncedLoadGeneExpression.cancel(); + debouncedLoadHeatmapExpression.cancel(); }; - }, [debouncedLoadGeneExpression]); + }, [debouncedLoadHeatmapExpression]); - const loadingGenes = useSelector((state) => state.genes.expression.full.loading); + const { + loading: downsampledLoading, + error: downsampledError, + cellIds: workerCellIds, + downsampleType, + lastFetchSettings, + } = useSelector((state) => state.genes.expression.downsampled); + + const downsampledMatrix = useSelector((state) => state.genes.expression.downsampled.matrix); const config = useSelector((state) => state.componentConfig[COMPONENT_TYPE]?.config) || {}; @@ -90,15 +100,19 @@ const HeatmapPlot = (props) => { } = useSelector((state) => state.genes.expression.full); // Create a stable reference that only changes when the heatmap's selected genes - // are loaded in the matrix (ignoring other genes added by other components) + // are loaded in the relevant matrix (full for small datasets, downsampled for large datasets) const heatmapGenesLoadedKey = useSelector((state) => { if (!selectedGenes?.length) return null; - const matrix = state.genes.expression.full.matrix; - // Check which of the heatmap's genes are currently loaded - const loadedCount = selectedGenes.filter((gene) => matrix.geneIsLoaded(gene)).length; - // Return a key that only changes when the loaded genes for heatmap actually change - const key = `${selectedGenes.length}_${loadedCount}`; - return key; + const sampleNode = state.cellSets.hierarchy?.find((node) => node.key === 'sample'); + const cells = sampleNode?.children?.reduce((sum, child) => { + const cellIds = state.cellSets.properties[child.key]?.cellIds; + return sum + (cellIds?.size || 0); + }, 0) || 0; + const mat = cells >= LARGE_DATASET_THRESHOLD + ? state.genes.expression.downsampled.matrix + : state.genes.expression.full.matrix; + const loadedCount = selectedGenes.filter((gene) => mat.geneIsLoaded(gene)).length; + return `${cells >= LARGE_DATASET_THRESHOLD ? 'd' : 'f'}_${selectedGenes.length}_${loadedCount}`; }); const { @@ -107,28 +121,15 @@ const HeatmapPlot = (props) => { const cellSets = useSelector(getCellSets()); - // Calculate nMarkerGenes based on dataset size and cluster count - const nMarkerGenes = useMemo(() => { - if (!cellSets?.properties || !cellSets?.hierarchy) { - console.log('[nMarkerGenes] No cellSets data available'); - return 5; // Default - } - - // Get total cell count from 'sample' hierarchy - const sampleNode = cellSets.hierarchy.find((node) => node.key === 'sample'); - const totalCells = sampleNode?.children?.reduce((sum, child) => { + const totalCells = useMemo(() => { + const sampleNode = cellSets.hierarchy?.find((node) => node.key === 'sample'); + return sampleNode?.children?.reduce((sum, child) => { const cellIds = cellSets.properties[child.key]?.cellIds; return sum + (cellIds?.size || 0); }, 0) || 0; + }, [cellSets.hierarchy, cellSets.properties]); - // Get cluster count from 'louvain' hierarchy - const louvainNode = cellSets.hierarchy.find((node) => node.key === 'louvain'); - const clusterCount = louvainNode?.children?.length || 0; - - const result = calculateNMarkerGenes(totalCells, clusterCount); - console.log('[nMarkerGenes] Calculated final result:', result); - return result; - }, [cellSets]); + const isLargeDataset = totalCells >= LARGE_DATASET_THRESHOLD; // Note: selectedPoints is not needed for vitessce heatmap as it's always 'All' const heatmapSettings = useSelector((state) => state.componentConfig[COMPONENT_TYPE]?.config, @@ -151,6 +152,40 @@ const HeatmapPlot = (props) => { const viewError = useSelector((state) => state.genes.expression.views[COMPONENT_TYPE]?.error); + // Map from cell ID → matrix column index for downsampled expression data + const cellIdToMatrixIndex = useMemo(() => { + if (!isLargeDataset || !workerCellIds?.length) return null; + return new Map(workerCellIds.map((id, i) => [id, i])); + }, [isLargeDataset, workerCellIds]); + + // Bucketed downsampling: re-sample display cells from worker-returned cells applying hidden sets. + // Deps intentionally exclude cellSets.properties / cellSets.hierarchy so that adding a custom + // cell set to the scratchpad does not trigger a re-run. + const bucketedDisplayCellIds = useMemo(() => { + if (!isLargeDataset || downsampleType !== 'bucketed' || !workerCellIds?.length) return null; + return computeBucketedDisplayCellIds( + heatmapSettings.selectedCellSet, + heatmapSettings.groupedTracks, + Array.from(cellSets.hidden || []), + cellSets, + workerCellIds, + ); + }, [ + isLargeDataset, + downsampleType, + workerCellIds, + cellSets.hidden, + heatmapSettings.selectedCellSet, + heatmapSettings.groupedTracks, + ]); + + const finalDisplayCellIds = useMemo(() => { + if (!isLargeDataset) return null; + if (downsampleType === 'bucketed') return bucketedDisplayCellIds; + if (downsampleType === 'precomputed') return workerCellIds; + return null; + }, [isLargeDataset, downsampleType, bucketedDisplayCellIds, workerCellIds]); + const updateCellCoordinates = (newView) => { if (cellHighlight && newView.projectFromId) { const [x, y] = newView.projectFromId(cellHighlight, geneHighlight); @@ -180,38 +215,25 @@ const HeatmapPlot = (props) => { }, [heatmapSettings]); useEffect(() => { - // Check if genes are currently being loaded - // For downsampled expressions, we just need to check fetchingGenes - const selectedGenesLoading = fetchingGenes - || (_.intersection(selectedGenes, loadingGenes).length > 0); + const selectedGenesLoading = isLargeDataset + ? downsampledLoading + : fetchingGenes || false; - // markerGenesLoading only happen on the first load - // selectedGenesLoading happens every time the selected genes are changed if (selectedGenesLoading || markerGenesLoading) { setIsHeatmapGenesLoading(true); return; } setIsHeatmapGenesLoading(false); - }, [selectedGenes, loadingGenes, markerGenesLoading, fetchingGenes]); + }, [markerGenesLoading, fetchingGenes, isLargeDataset, downsampledLoading]); + // Small dataset: generate heatmap from full expression matrix useEffect(() => { - if ( - !selectedGenes?.length - || !cellSets.hierarchy?.length - ) { return; } - // Check that the expression data has actually been loaded into the matrix - // Just having geneIndexes isn't enough - we need the rawGeneExpressions data - const [cellCount, geneCount] = matrix?.rawGeneExpressions?.size?.() || [0, 0]; - if (!geneCount || geneCount === 0) { - // Data not ready yet, wait for the expression values to be populated - return; - } + if (isLargeDataset) return; + if (!selectedGenes?.length || !cellSets.hierarchy?.length) return; - // Selected genes is not contained in heatmap settings for the - // data exploration marker heatmap, so must be passed spearatedly. - // Trying to assign it to heatmapSettings will throw an error because - // heatmapSettings is is frozen in redux by immer. + const [, geneCount] = matrix?.rawGeneExpressions?.size?.() || [0, 0]; + if (!geneCount) return; const data = generateVitessceData( selectedTracks, @@ -222,6 +244,7 @@ const HeatmapPlot = (props) => { ); setHeatmapData(data); }, [ + isLargeDataset, selectedGenes, selectedTracks, heatmapGenesLoadedKey, @@ -232,6 +255,68 @@ const HeatmapPlot = (props) => { heatmapSettings?.groupedTracks, ]); + // Large dataset: generate heatmap from downsampled expression matrix. + // cellSets.properties and cellSets.hierarchy are included so that track labels/colours + // stay correct; the expensive getBuckets computation is avoided because finalDisplayCellIds + // is pre-computed and passed in directly. + useEffect(() => { + if (!isLargeDataset) return; + if (!selectedGenes?.length || !cellSets.hierarchy?.length) return; + if (downsampledLoading || finalDisplayCellIds === null) return; + + // Don't render until all selected genes are present in the downsampled matrix. + // This prevents a flash of wrong data when genes are added but haven't been fetched yet. + const allGenesLoaded = selectedGenes.every((gene) => downsampledMatrix?.geneIsLoaded(gene)); + if (!allGenesLoaded) return; + + // Prevent rendering stale data while a new work request is in-flight. + // The matrix was built with the settings in lastFetchSettings; if those differ from + // the current heatmap settings, the new request hasn't completed yet. + if (lastFetchSettings) { + const matrixMatchesSettings = ( + lastFetchSettings.selectedCellSet === heatmapSettings.selectedCellSet + && _.isEqual(lastFetchSettings.groupedTracks, heatmapSettings.groupedTracks) + ); + if (!matrixMatchesSettings) return; + } + + // All cells hidden — generate empty data to show the "unhide" message + if (finalDisplayCellIds.length === 0) { + setHeatmapData(generateVitessceData( + selectedTracks, downsampledMatrix, selectedGenes, cellSets, heatmapSettings, + [], cellIdToMatrixIndex, + )); + return; + } + + const [, geneCount] = downsampledMatrix?.rawGeneExpressions?.size?.() || [0, 0]; + if (!geneCount) return; + + const data = generateVitessceData( + selectedTracks, + downsampledMatrix, + selectedGenes, + cellSets, + heatmapSettings, + finalDisplayCellIds, + cellIdToMatrixIndex, + ); + setHeatmapData(data); + }, [ + isLargeDataset, + downsampledLoading, + finalDisplayCellIds, + selectedGenes, + selectedTracks, + cellIdToMatrixIndex, + lastFetchSettings, + cellSets.properties, + cellSets.hierarchy, + cellSets.hidden, + heatmapSettings?.selectedCellSet, + heatmapSettings?.groupedTracks, + ]); + useConditionalEffect(() => { if ( !cellSets.accessible @@ -247,7 +332,7 @@ const HeatmapPlot = (props) => { dispatch(loadMarkerGenes( experimentId, COMPONENT_TYPE, - { numGenes: nMarkerGenes, selectedCellSet }, + { numGenes: NUM_MARKER_GENES, selectedCellSet }, )); }, [ louvainClustersResolution, @@ -256,7 +341,6 @@ const HeatmapPlot = (props) => { groupedCellSets, ]); - // Only fetch gene expression if selectedGenes or selectedCellSet change useConditionalEffect( () => { if ( @@ -265,21 +349,26 @@ const HeatmapPlot = (props) => { || !heatmapSettings.groupedTracks || !heatmapSettings.selectedCellSet || selectedGenes.length === 0 - || fetchingGenes ) { return; } - // Only fetch if selectedGenes or selectedCellSet changed - debouncedLoadGeneExpression( + debouncedLoadHeatmapExpression( experimentId, selectedGenes, - COMPONENT_TYPE, + { + selectedCellSet: heatmapSettings.selectedCellSet, + groupedTracks: heatmapSettings.groupedTracks, + hiddenCellSets: Array.from(cellSets.hidden || []), + plotUuid: COMPONENT_TYPE, + }, ); }, [ louvainClustersResolution, cellSets.accessible, heatmapSettings?.selectedCellSet, + heatmapSettings?.groupedTracks, selectedGenes, + cellSets.hidden, ], ); @@ -294,10 +383,11 @@ const HeatmapPlot = (props) => { [], ); - if (markerGenesLoadingError || expressionDataError || viewError) { + const expressionError = isLargeDataset ? downsampledError : expressionDataError; + if (markerGenesLoadingError || expressionError || viewError) { return ( { if (markerGenesLoadingError) { const { selectedCellSet } = heatmapSettings; @@ -306,15 +396,22 @@ const HeatmapPlot = (props) => { experimentId, COMPONENT_TYPE, { - numGenes: nMarkerGenes, + numGenes: NUM_MARKER_GENES, selectedCellSet, }, )); } - if ((expressionDataError || viewError) && selectedGenes.length > 0) { - debouncedLoadGeneExpression( - experimentId, selectedGenes, COMPONENT_TYPE, true, + if ((expressionError || viewError) && selectedGenes.length > 0) { + debouncedLoadHeatmapExpression( + experimentId, + selectedGenes, + { + selectedCellSet: heatmapSettings.selectedCellSet, + groupedTracks: heatmapSettings.groupedTracks, + hiddenCellSets: Array.from(cellSets.hidden || []), + plotUuid: COMPONENT_TYPE, + }, ); } }} @@ -333,7 +430,8 @@ const HeatmapPlot = (props) => { } // Also check if the expression matrix has actual gene data loaded - if (!matrix?.geneIndexes || Object.keys(matrix.geneIndexes).length === 0) { + const activeMatrix = isLargeDataset ? downsampledMatrix : matrix; + if (!activeMatrix?.geneIndexes || Object.keys(activeMatrix.geneIndexes).length === 0) { return (
@@ -426,7 +524,14 @@ const HeatmapPlot = (props) => { cellId={cellHighlight} geneName={geneHighlight} geneExpression={ - expressionMatrix.getRawExpression(geneHighlight, [parseInt(cellHighlight, 10)]) + isLargeDataset + ? cellIdToMatrixIndex?.has(parseInt(cellHighlight, 10)) + ? downsampledMatrix.getRawExpression?.( + geneHighlight, + [cellIdToMatrixIndex.get(parseInt(cellHighlight, 10))], + ) + : undefined + : expressionMatrix.getRawExpression(geneHighlight, [parseInt(cellHighlight, 10)]) } coordinates={cellCoordinatesRef.current} /> diff --git a/src/components/data-processing/ConfigureEmbedding/ConfigureEmbedding.jsx b/src/components/data-processing/ConfigureEmbedding/ConfigureEmbedding.jsx index 7f2a2d3fe9..2c6c1e484f 100644 --- a/src/components/data-processing/ConfigureEmbedding/ConfigureEmbedding.jsx +++ b/src/components/data-processing/ConfigureEmbedding/ConfigureEmbedding.jsx @@ -436,7 +436,7 @@ const ConfigureEmbedding = (props) => { debounceSave(activePlotUuid); } } - }, [activePlotUuid, currentPlot?.plotType, cellSets?.accessible]); + }, [activePlotUuid, currentPlot?.plotType, cellSets?.accessible, !!selectedConfig]); useEffect(() => { // if we change a plot and the config is not saved yet diff --git a/src/components/data-processing/DataIntegration/DataIntegration.jsx b/src/components/data-processing/DataIntegration/DataIntegration.jsx index ba168c3a89..9abdbe1c50 100644 --- a/src/components/data-processing/DataIntegration/DataIntegration.jsx +++ b/src/components/data-processing/DataIntegration/DataIntegration.jsx @@ -314,7 +314,7 @@ const DataIntegration = (props) => { debounceSave(activePlotUuid); } } - }, [activePlotUuid, activePlotType, cellSets?.accessible]); + }, [activePlotUuid, activePlotType, cellSets?.accessible, !!selectedConfig]); const completedSteps = useSelector(getBackendStatus(experimentId)) .status?.pipeline?.completedSteps; diff --git a/src/components/plots/helpers/heatmap/vega/generateVegaData.js b/src/components/plots/helpers/heatmap/vega/generateVegaData.js index 6ef52df969..144f867f91 100644 --- a/src/components/plots/helpers/heatmap/vega/generateVegaData.js +++ b/src/components/plots/helpers/heatmap/vega/generateVegaData.js @@ -7,6 +7,8 @@ import getHeatmapCellOrder, { const generateVegaData = ( expressionMatrix, heatmapSettings, cellSets, + preComputedCellOrder = null, + cellIdToMatrixIndex = null, ) => { const { selectedGenes, selectedTracks, guardLines, selectedCellSet, selectedPoints, groupedTracks, @@ -16,7 +18,7 @@ const generateVegaData = ( // Compute which cell sets should be hidden based on selectedPoints const hiddenCellSets = computeHiddenCellSets(selectedPoints, cellSets); - const cellOrder = getHeatmapCellOrder( + const cellOrder = preComputedCellOrder ?? getHeatmapCellOrder( selectedCellSet, groupedTracks, hiddenCellSets, @@ -33,7 +35,7 @@ const generateVegaData = ( }; data.geneExpressionsData = generateVegaGeneExpressionsData( - cellOrder, selectedGenes, expressionMatrix, heatmapSettings, + cellOrder, selectedGenes, expressionMatrix, heatmapSettings, cellIdToMatrixIndex, ); const trackData = trackOrder.map( diff --git a/src/components/plots/helpers/heatmap/vega/utils/generateVegaGeneExpressionsData.js b/src/components/plots/helpers/heatmap/vega/utils/generateVegaGeneExpressionsData.js index ab16a9df52..341cb8a645 100644 --- a/src/components/plots/helpers/heatmap/vega/utils/generateVegaGeneExpressionsData.js +++ b/src/components/plots/helpers/heatmap/vega/utils/generateVegaGeneExpressionsData.js @@ -7,7 +7,7 @@ const cartesian = (...array) => ( ); const generateVegaGeneExpressionsData = ( - cellOrder, geneOrder, expressionMatrix, heatmapSettings, + cellOrder, geneOrder, expressionMatrix, heatmapSettings, cellIdToMatrixIndex = null, ) => { const { expressionValue, truncatedValues } = heatmapSettings; @@ -17,19 +17,27 @@ const generateVegaGeneExpressionsData = ( return; } + // For downsampled matrices, cellOrder contains cell IDs but the matrix is indexed + // by position in workerCellIds. cellIdToMatrixIndex maps cell ID → column index. + const matrixIndices = cellIdToMatrixIndex + ? cellOrder.map((id) => cellIdToMatrixIndex.get(id)) + : cellOrder; + // Preload all genes so that their arrays are generated only once const preloadedExpressions = {}; geneOrder.forEach((gene) => { if (expressionValue === 'zScore') { - preloadedExpressions[gene] = { zScore: expressionMatrix.getZScore(gene, cellOrder) }; + preloadedExpressions[gene] = { zScore: expressionMatrix.getZScore(gene, matrixIndices) }; return; } - const geneExpression = { rawExpression: expressionMatrix.getRawExpression(gene, cellOrder) }; + const geneExpression = { + rawExpression: expressionMatrix.getRawExpression(gene, matrixIndices), + }; if (truncatedValues) { geneExpression.truncatedExpression = expressionMatrix.getTruncatedExpression( - gene, cellOrder, + gene, matrixIndices, ); } diff --git a/src/components/plots/helpers/heatmap/vitessce/generateVitessceData.js b/src/components/plots/helpers/heatmap/vitessce/generateVitessceData.js index 0ed42b1012..54e84788f1 100644 --- a/src/components/plots/helpers/heatmap/vitessce/generateVitessceData.js +++ b/src/components/plots/helpers/heatmap/vitessce/generateVitessceData.js @@ -5,14 +5,16 @@ import getHeatmapCellOrder from 'utils/work/getHeatmapCellOrder'; const generateVitessceData = ( selectedTracks, expressionMatrix, selectedGenes, cellSets, heatmapSettings, + preComputedCellOrder = null, + cellIdToMatrixIndex = null, ) => { - // Compute cellOrder internally based on heatmap settings + // Compute cellOrder internally based on heatmap settings, or use the pre-computed value + // (used for large datasets where the cell order is determined by the worker response). const { selectedCellSet = 'louvain', groupedTracks = [], } = heatmapSettings || {}; - // Use only user-hidden cells (selectedPoints is always 'All' for vitessce) - const cellOrder = getHeatmapCellOrder( + const cellOrder = preComputedCellOrder ?? getHeatmapCellOrder( selectedCellSet, groupedTracks, cellSets.hidden || [], @@ -27,6 +29,7 @@ const generateVitessceData = ( cellOrder, selectedGenes, expressionMatrix, + cellIdToMatrixIndex, ); const metadataTracksLabels = selectedTracks diff --git a/src/components/plots/helpers/heatmap/vitessce/utils/generateVitessceHeatmapExpressionsMatrix.js b/src/components/plots/helpers/heatmap/vitessce/utils/generateVitessceHeatmapExpressionsMatrix.js index 5f090ca4ce..ee13fa6b7c 100644 --- a/src/components/plots/helpers/heatmap/vitessce/utils/generateVitessceHeatmapExpressionsMatrix.js +++ b/src/components/plots/helpers/heatmap/vitessce/utils/generateVitessceHeatmapExpressionsMatrix.js @@ -5,16 +5,24 @@ const scaledTo255 = (rowOfExpressions, min, max) => ( rowOfExpressions.map((value) => convertRange(value, [min, max], [0, 255])) ); -const generateVitessceHeatmapExpressionsMatrix = (cellOrder, geneOrder, expressionMatrix) => { +const generateVitessceHeatmapExpressionsMatrix = ( + cellOrder, geneOrder, expressionMatrix, cellIdToMatrixIndex = null, +) => { const geneExpressionsDataMatrix = []; + // For downsampled matrices, cellOrder contains cell IDs, not matrix indices. + // cellIdToMatrixIndex maps cell ID → column index in the matrix. + const matrixIndices = cellIdToMatrixIndex + ? cellOrder.map((id) => cellIdToMatrixIndex.get(id)) + : cellOrder; + geneOrder.forEach((gene) => { const isLoaded = expressionMatrix.geneIsLoaded(gene); if (!isLoaded) { return; } - const truncatedExpression = expressionMatrix.getTruncatedExpression(gene, cellOrder); + const truncatedExpression = expressionMatrix.getTruncatedExpression(gene, matrixIndices); const { truncatedMin, truncatedMax } = expressionMatrix.getStats(gene); diff --git a/src/pages/experiments/[experimentId]/plots-and-tables/marker-heatmap/index.jsx b/src/pages/experiments/[experimentId]/plots-and-tables/marker-heatmap/index.jsx index e28cd362ec..694f00e5af 100644 --- a/src/pages/experiments/[experimentId]/plots-and-tables/marker-heatmap/index.jsx +++ b/src/pages/experiments/[experimentId]/plots-and-tables/marker-heatmap/index.jsx @@ -16,7 +16,6 @@ import 'vega-webgl-renderer'; import ExpressionMatrix from 'utils/ExpressionMatrix/ExpressionMatrix'; import { getCellSets, getCellSetsHierarchyByKeys } from 'redux/selectors'; -import calculateNMarkerGenes from 'utils/calculateNMarkerGenes'; import HeatmapGroupBySettings from 'components/data-exploration/heatmap/HeatmapGroupBySettings'; import HeatmapMetadataTracksSettings from 'components/data-exploration/heatmap/HeatmapMetadataTrackSettings'; @@ -28,9 +27,11 @@ import Header from 'components/Header'; import PlotContainer from 'components/plots/PlotContainer'; import { generateSpec } from 'utils/plotSpecs/generateHeatmapSpec'; import { - loadGeneExpression, + loadHeatmapExpression, loadMarkerGenes, } from 'redux/actions/genes'; +import { LARGE_DATASET_THRESHOLD } from 'redux/actions/genes/loadHeatmapExpression'; +import { computeBucketedDisplayCellIds, computeHiddenCellSets } from 'utils/work/getHeatmapCellOrder'; import loadGeneList from 'redux/actions/genes/loadGeneList'; import { loadCellSets } from 'redux/actions/cellSets'; import PlatformError from 'components/PlatformError'; @@ -48,6 +49,7 @@ import useConditionalEffect from 'utils/customHooks/useConditionalEffect'; const { Panel } = Collapse; const plotUuid = 'markerHeatmapPlotMain'; const plotType = 'markerHeatmap'; +const NUM_MARKER_GENES = 5; const MarkerHeatmap = ({ experimentId }) => { const dispatch = useDispatch(); @@ -72,6 +74,14 @@ const MarkerHeatmap = ({ experimentId }) => { (state) => state.genes.expression.views[plotUuid], ) || {}; + const { + loading: downsampledLoading, + error: downsampledError, + cellIds: workerCellIds, + downsampleType, + } = useSelector((state) => state.genes.expression.downsampled); + const downsampledMatrix = useSelector((state) => state.genes.expression.downsampled.matrix); + const cellSets = useSelector(getCellSets()); const { hierarchy } = cellSets; @@ -95,25 +105,48 @@ const MarkerHeatmap = ({ experimentId }) => { .configureEmbedding?.clusteringSettings.methodSettings.louvain.resolution, ) || false; - // Calculate nMarkerGenes based on dataset size and cluster count - const nMarkerGenes = React.useMemo(() => { - if (!cellSets?.properties || !cellSets?.hierarchy) { - return 5; // Default - } - - // Get total cell count from 'sample' hierarchy - const sampleNode = cellSets.hierarchy.find((node) => node.key === 'sample'); - const totalCells = sampleNode?.children?.reduce((sum, child) => { + const totalCells = React.useMemo(() => { + const sampleNode = cellSets.hierarchy?.find((node) => node.key === 'sample'); + return sampleNode?.children?.reduce((sum, child) => { const cellIds = cellSets.properties[child.key]?.cellIds; return sum + (cellIds?.size || 0); }, 0) || 0; + }, [cellSets.hierarchy, cellSets.properties]); + + const isLargeDataset = totalCells >= LARGE_DATASET_THRESHOLD; + + // Map from cell ID → matrix column index for downsampled expression data + const cellIdToMatrixIndex = React.useMemo(() => { + if (!isLargeDataset || !workerCellIds?.length) return null; + return new Map(workerCellIds.map((id, i) => [id, i])); + }, [isLargeDataset, workerCellIds]); + + // Bucketed downsampling: re-sample display cells from worker-returned cells applying hidden sets + const bucketedDisplayCellIds = React.useMemo(() => { + if (!isLargeDataset || downsampleType !== 'bucketed' || !workerCellIds?.length) return null; + const hiddenCellSets = computeHiddenCellSets(config?.selectedPoints, cellSets); + return computeBucketedDisplayCellIds( + config?.selectedCellSet, + config?.groupedTracks, + hiddenCellSets, + cellSets, + workerCellIds, + ); + }, [ + isLargeDataset, + downsampleType, + workerCellIds, + config?.selectedPoints, + config?.selectedCellSet, + config?.groupedTracks, + ]); - // Get cluster count from 'louvain' hierarchy - const louvainNode = cellSets.hierarchy.find((node) => node.key === 'louvain'); - const clusterCount = louvainNode?.children?.length || 0; - - return calculateNMarkerGenes(totalCells, clusterCount); - }, [cellSets]); + const finalDisplayCellIds = React.useMemo(() => { + if (!isLargeDataset) return null; + if (downsampleType === 'bucketed') return bucketedDisplayCellIds; + if (downsampleType === 'precomputed') return workerCellIds; + return null; + }, [isLargeDataset, downsampleType, bucketedDisplayCellIds, workerCellIds]); useEffect(() => { if (!louvainClustersResolution) dispatch(loadProcessingSettings(experimentId)); @@ -147,9 +180,13 @@ const MarkerHeatmap = ({ experimentId }) => { }, )); } else if (updatesToDispatch.selectedGenes) { - dispatch( - loadGeneExpression(experimentId, updatesToDispatch.selectedGenes, plotUuid), - ); + const hiddenCellSets = computeHiddenCellSets(config?.selectedPoints, cellSets); + dispatch(loadHeatmapExpression(experimentId, updatesToDispatch.selectedGenes, { + selectedCellSet: config?.selectedCellSet, + groupedTracks: config?.groupedTracks, + hiddenCellSets, + plotUuid, + })); } }; @@ -166,12 +203,12 @@ const MarkerHeatmap = ({ experimentId }) => { // If the plot has never been loaded (so selectedGenes is null), then load the marker genes // Only auto-load on initial render (null), not when user clears genes (empty array) useEffect(() => { - if (config?.selectedGenes === null && nMarkerGenes) { + if (config?.selectedGenes === null) { dispatch(loadMarkerGenes( experimentId, plotUuid, { - numGenes: nMarkerGenes, + numGenes: NUM_MARKER_GENES, groupedTracks: config.groupedTracks, selectedCellSet: config.selectedCellSet, selectedPoints: config.selectedPoints, @@ -179,7 +216,6 @@ const MarkerHeatmap = ({ experimentId }) => { )); } }, [ - nMarkerGenes, JSON.stringify(config?.groupedTracks), config?.selectedCellSet, config?.selectedPoints, @@ -209,25 +245,28 @@ const MarkerHeatmap = ({ experimentId }) => { if (!expectedConditions) { return; } - dispatch(loadGeneExpression(experimentId, loadedGenes, plotUuid)); + const hiddenCellSets = computeHiddenCellSets(config.selectedPoints, cellSets); + dispatch(loadHeatmapExpression(experimentId, loadedGenes, { + selectedCellSet: config.selectedCellSet, + groupedTracks: config.groupedTracks, + hiddenCellSets, + plotUuid, + })); }, [ loadedGenes, config?.selectedCellSet, + config?.groupedTracks, config?.selectedPoints, hierarchy, cellSets.accessible, louvainClustersResolution, ]); + // Small dataset: generate heatmap from full expression matrix useEffect(() => { - // Don't create spec while marker genes are loading (prevents stale spec recreation) - if (markerGenesLoading || expressionFetching) { - return; - } - + if (isLargeDataset) return; + if (markerGenesLoading || expressionFetching) return; - // Check preconditions: data is loaded and ready - // cellOrder will be computed internally by generateVegaData if ( !cellSets.accessible || !cellSets.hierarchy?.length @@ -238,32 +277,23 @@ const MarkerHeatmap = ({ experimentId }) => { return; } - // Check preconditions: no errors and data is not fetching if (expressionError || markerGenesLoadingError) { return; } - // Check that the expression data has actually been loaded into the matrix - // Just having geneIndexes isn't enough - we need the rawGeneExpressions data - const [cellCount, geneCount] = rawMatrix?.rawGeneExpressions?.size?.() || [0, 0]; + const [, geneCount] = rawMatrix?.rawGeneExpressions?.size?.() || [0, 0]; if (!geneCount || geneCount === 0) { - // Data not ready yet, wait for the expression values to be populated return; } - // Verify that the matrix has ALL the genes we need to render - // Note: matrix can have MORE genes than loadedGenes (it accumulates genes from previous operations) - // We only care that all genes in loadedGenes are present in the matrix const matrixGeneIndexes = rawMatrix?.geneIndexes || {}; const allGenesAvailable = loadedGenes.every((gene) => matrixGeneIndexes[gene] !== undefined); if (!allGenesAvailable) { - // Not all genes are in the matrix yet, wait for expression data to be fully loaded return; } - // Reconstruct ExpressionMatrix from plain object (lost prototype after Redux hydration) const matrix = new ExpressionMatrix(); if (rawMatrix?.geneIndexes && rawMatrix?.rawGeneExpressions && rawMatrix?.stats) { matrix.geneIndexes = rawMatrix.geneIndexes; @@ -271,9 +301,6 @@ const MarkerHeatmap = ({ experimentId }) => { matrix.stats = rawMatrix.stats; } - // Pass loadedGenes as selectedGenes to vega data generator - // This ensures we render the loaded genes instead of config.selectedGenes which may be empty - // generateVegaData will internally compute cellOrder based on config const vegaData = generateVegaData(matrix, { ...config, selectedGenes: loadedGenes }, cellSets); const { cellOrder: computedCellOrder } = vegaData; setCellOrder(computedCellOrder); @@ -300,7 +327,116 @@ const MarkerHeatmap = ({ experimentId }) => { spec.marks.push(extraMarks); setVegaSpec(spec); - }, [selectedTracks, loadedGenes, selectedCellSetConfig, config?.selectedPoints, config?.groupedTracks, expressionError, cellSets.accessible, cellSets.hierarchy, cellSets.hidden, hierarchy, markerGenesLoadingError, markerGenesLoading, rawMatrix, expressionFetching]); + }, [ + isLargeDataset, + selectedTracks, + loadedGenes, + selectedCellSetConfig, + config?.selectedPoints, + config?.groupedTracks, + expressionError, + cellSets.accessible, + cellSets.hierarchy, + cellSets.hidden, + hierarchy, + markerGenesLoadingError, + markerGenesLoading, + rawMatrix, + expressionFetching, + ]); + + // Large dataset: generate heatmap from downsampled expression matrix + useEffect(() => { + if (!isLargeDataset) return; + if (markerGenesLoading || expressionFetching) return; + + if ( + !cellSets.accessible + || !cellSets.hierarchy?.length + || !loadedGenes?.length + || !hierarchy?.length + || downsampledLoading + || !finalDisplayCellIds?.length + ) { + return; + } + + if (downsampledError || markerGenesLoadingError) { + return; + } + + const [, geneCount] = downsampledMatrix?.rawGeneExpressions?.size?.() || [0, 0]; + + if (!geneCount || geneCount === 0) { + return; + } + + const matrixGeneIndexes = downsampledMatrix?.geneIndexes || {}; + const allGenesAvailable = loadedGenes.every((gene) => matrixGeneIndexes[gene] !== undefined); + + if (!allGenesAvailable) { + return; + } + + const matrix = new ExpressionMatrix(); + if (downsampledMatrix?.geneIndexes && downsampledMatrix?.rawGeneExpressions && downsampledMatrix?.stats) { + matrix.geneIndexes = downsampledMatrix.geneIndexes; + matrix.rawGeneExpressions = downsampledMatrix.rawGeneExpressions; + matrix.stats = downsampledMatrix.stats; + } + + const vegaData = generateVegaData( + matrix, + { ...config, selectedGenes: loadedGenes }, + cellSets, + finalDisplayCellIds, + cellIdToMatrixIndex, + ); + const { cellOrder: computedCellOrder } = vegaData; + setCellOrder(computedCellOrder); + const spec = generateSpec(config, 'Cluster ID', vegaData, config.showGeneLabels); + + spec.description = 'Marker heatmap'; + + const extraMarks = { + type: 'rule', + from: { data: 'clusterSeparationLines' }, + encode: { + enter: { + stroke: { value: 'white' }, + }, + update: { + x: { scale: 'x', field: 'data' }, + y: 0, + y2: { field: { group: 'height' } }, + strokeWidth: { value: 1 }, + strokeOpacity: { value: 1 }, + }, + }, + }; + spec.marks.push(extraMarks); + + setVegaSpec(spec); + }, [ + isLargeDataset, + downsampledLoading, + finalDisplayCellIds, + selectedTracks, + loadedGenes, + selectedCellSetConfig, + config?.selectedPoints, + config?.groupedTracks, + downsampledError, + cellIdToMatrixIndex, + cellSets.accessible, + cellSets.hierarchy, + cellSets.hidden, + hierarchy, + markerGenesLoadingError, + markerGenesLoading, + downsampledMatrix, + expressionFetching, + ]); useEffect(() => { dispatch(loadGeneList(experimentId)); @@ -355,16 +491,27 @@ const MarkerHeatmap = ({ experimentId }) => { // Update config with the new gene order first (single source of truth) dispatch(updatePlotConfig(plotUuid, { selectedGenes: newGenes })); // Then load gene expression with the new order - dispatch(loadGeneExpression(experimentId, newGenes, plotUuid)); - }, [experimentId, plotUuid, dispatch]); + const hiddenCellSets = computeHiddenCellSets(config?.selectedPoints, cellSets); + dispatch(loadHeatmapExpression(experimentId, newGenes, { + selectedCellSet: config?.selectedCellSet, + groupedTracks: config?.groupedTracks, + hiddenCellSets, + plotUuid, + })); + }, [experimentId, plotUuid, dispatch, config?.selectedCellSet, config?.groupedTracks, config?.selectedPoints, cellSets]); const onGenesSelect = (genes) => { const allGenes = _.uniq([...loadedGenes, ...genes]); if (_.isEqual(allGenes, loadedGenes)) return; - // Load the selected genes (updates genes.expression.views) - dispatch(loadGeneExpression(experimentId, allGenes, plotUuid)); + const hiddenCellSets = computeHiddenCellSets(config?.selectedPoints, cellSets); + dispatch(loadHeatmapExpression(experimentId, allGenes, { + selectedCellSet: config?.selectedCellSet, + groupedTracks: config?.groupedTracks, + hiddenCellSets, + plotUuid, + })); }; const onReset = () => { @@ -372,7 +519,7 @@ const MarkerHeatmap = ({ experimentId }) => { experimentId, plotUuid, { - numGenes: nMarkerGenes, + numGenes: NUM_MARKER_GENES, groupedTracks: config.groupedTracks, selectedCellSet: config.selectedCellSet, selectedPoints: config.selectedPoints, @@ -513,15 +660,20 @@ const MarkerHeatmap = ({ experimentId }) => { ); } - if (expressionError) { + const expressionErr = isLargeDataset ? downsampledError : expressionError; + if (expressionErr) { return ( { - dispatch( - loadGeneExpression(experimentId, loadedGenes, plotUuid), - ); + const hiddenCellSets = computeHiddenCellSets(config?.selectedPoints, cellSets); + dispatch(loadHeatmapExpression(experimentId, loadedGenes, { + selectedCellSet: config?.selectedCellSet, + groupedTracks: config?.groupedTracks, + hiddenCellSets, + plotUuid, + })); }} /> ); @@ -550,11 +702,12 @@ const MarkerHeatmap = ({ experimentId }) => { ); } + const loadingExpr = isLargeDataset ? downsampledLoading : expressionFetching; if ( !config || !cellSets.accessible || markerGenesLoading - || expressionFetching + || loadingExpr ) { return (); } diff --git a/src/redux/actionTypes/genes.js b/src/redux/actionTypes/genes.js index 7b62091311..84ac16252e 100644 --- a/src/redux/actionTypes/genes.js +++ b/src/redux/actionTypes/genes.js @@ -60,9 +60,25 @@ const MARKER_GENES_LOADED = `${GENES}/markerGenesLoaded`; */ const MARKER_GENES_ERROR = `${GENES}/markerGenesError`; +/** + * Turns on the loading state for heatmap expression data (downsampled cases). + */ +const HEATMAP_EXPRESSION_LOADING = `${GENES}/heatmapExpressionLoading`; + +/** + * Sets the state of the heatmap downsampled expression store to successfully loaded. + */ +const HEATMAP_EXPRESSION_LOADED = `${GENES}/heatmapExpressionLoaded`; + +/** + * Sets an error condition for heatmap downsampled expression data. + */ +const HEATMAP_EXPRESSION_ERROR = `${GENES}/heatmapExpressionError`; + export { GENES_PROPERTIES_LOADING, GENES_PROPERTIES_LOADED_PAGINATED, GENES_PROPERTIES_ERROR, GENES_SELECT, GENES_DESELECT, MARKER_GENES_LOADING, MARKER_GENES_LOADED, MARKER_GENES_ERROR, GENES_EXPRESSION_LOADING, GENES_EXPRESSION_LOADED, GENES_EXPRESSION_ERROR, + HEATMAP_EXPRESSION_LOADING, HEATMAP_EXPRESSION_LOADED, HEATMAP_EXPRESSION_ERROR, }; diff --git a/src/redux/actions/genes/index.js b/src/redux/actions/genes/index.js index f868f286de..96a984f711 100644 --- a/src/redux/actions/genes/index.js +++ b/src/redux/actions/genes/index.js @@ -2,10 +2,12 @@ import loadGeneExpression from './loadGeneExpression'; import changeGeneSelection from './changeGeneSelection'; import loadPaginatedGeneProperties from './loadPaginatedGeneProperties'; import loadMarkerGenes from './loadMarkerGenes'; +import loadHeatmapExpression from './loadHeatmapExpression'; export { loadGeneExpression, changeGeneSelection, loadPaginatedGeneProperties, loadMarkerGenes, + loadHeatmapExpression, }; diff --git a/src/redux/actions/genes/loadHeatmapExpression.js b/src/redux/actions/genes/loadHeatmapExpression.js new file mode 100644 index 0000000000..61250ff354 --- /dev/null +++ b/src/redux/actions/genes/loadHeatmapExpression.js @@ -0,0 +1,177 @@ +import _ from 'lodash'; +import { SparseMatrix } from 'mathjs'; + +import { + HEATMAP_EXPRESSION_LOADING, + HEATMAP_EXPRESSION_LOADED, + HEATMAP_EXPRESSION_ERROR, +} from 'redux/actionTypes/genes'; + +import loadGeneExpression from 'redux/actions/genes/loadGeneExpression'; +import fetchWork from 'utils/work/fetchWork'; +import getTimeoutForWorkerTask from 'utils/getTimeoutForWorkerTask'; +import upperCaseArray from 'utils/upperCaseArray'; +import getHeatmapCellOrder, { getBuckets } from 'utils/work/getHeatmapCellOrder'; +import { getCellSets } from 'redux/selectors'; + +const LARGE_DATASET_THRESHOLD = 50000; + +const getTotalCells = (cellSets) => { + const sampleNode = cellSets.hierarchy?.find((node) => node.key === 'sample'); + return sampleNode?.children?.reduce((sum, child) => { + const cellIds = cellSets.properties[child.key]?.cellIds; + return sum + (cellIds?.size || 0); + }, 0) || 0; +}; + +const findLoadedGenesInDownsampled = (matrix, genes) => { + const storedGenes = matrix.getStoredGenes(); + const genesToLoad = [...genes].filter( + (gene) => !new Set(upperCaseArray(storedGenes)).has(gene.toUpperCase()), + ); + const genesAlreadyLoaded = storedGenes.filter( + (gene) => upperCaseArray(genes).includes(gene.toUpperCase()), + ); + return { genesToLoad, genesAlreadyLoaded }; +}; + +const settingsHaveChanged = (lastFetchSettings, newSettings) => { + if (!lastFetchSettings) return true; + if (lastFetchSettings.downsampleType !== newSettings.downsampleType) return true; + if (newSettings.downsampleType === 'bucketed') { + return ( + lastFetchSettings.selectedCellSet !== newSettings.selectedCellSet + || !_.isEqual(lastFetchSettings.groupedTracks, newSettings.groupedTracks) + ); + } + // precomputed: compare by serialized cell IDs + return lastFetchSettings.cellIdsKey !== newSettings.cellIdsKey; +}; + +/** + * Loads heatmap expression data, routing to the appropriate strategy based on dataset size. + * + * Small datasets (< 50k cells): delegates to loadGeneExpression (full matrix). + * Bucketed (≥ 50k, capped bucket total < 50k): + * sends { downsampleSettings: { selectedCellSet, groupedTracks } }. + * The worker samples up to 1000 cells per bucket; + * client does a second-pass proportional downsampling. + * Precomputed (capped bucket total ≥ 50k): + * sends { downsampleSettings: { cellIds } } with an explicit + * pre-computed list of cells (up to 5000, hidden sets factored in). + * + * @param {string} experimentId + * @param {string[]} genes + * @param {object} options + * @param {string} options.selectedCellSet + * @param {string[]} options.groupedTracks + * @param {string[]} options.hiddenCellSets - computed by computeHiddenCellSets + * @param {string} options.plotUuid - used only for small dataset delegation + */ +const loadHeatmapExpression = ( + experimentId, + genes, + { + selectedCellSet, groupedTracks, hiddenCellSets, plotUuid, + }, +) => async (dispatch, getState) => { + if (!genes?.length) return null; + + const state = getState(); + const cellSets = getCellSets()(state); + + if (!cellSets.accessible) return null; + + const totalCells = getTotalCells(cellSets); + + // ── Small dataset: use full expression matrix ──────────────────────────── + if (totalCells < LARGE_DATASET_THRESHOLD) { + return dispatch(loadGeneExpression(experimentId, genes, plotUuid)); + } + + // ── Large dataset: determine bucketed vs precomputed ──────────────────────────── + const { buckets } = getBuckets(selectedCellSet, groupedTracks, [], cellSets); + const cappedTotal = buckets.reduce((sum, b) => sum + Math.min(b.size, 1000), 0); + const isBucketed = cappedTotal < LARGE_DATASET_THRESHOLD; + const downsampleType = isBucketed ? 'bucketed' : 'precomputed'; + + let cellIdsToRequest = null; + if (!isBucketed) { + cellIdsToRequest = getHeatmapCellOrder( + selectedCellSet, groupedTracks, hiddenCellSets, cellSets, 5000, + ); + } + + const newSettings = { + downsampleType, + selectedCellSet, + groupedTracks, + cellIdsKey: cellIdsToRequest ? cellIdsToRequest.join(',') : null, + }; + + const { downsampled } = state.genes.expression; + + // Don't issue a new request if already loading + if (downsampled.loading) return null; + + const changed = settingsHaveChanged(downsampled.lastFetchSettings, newSettings); + + const { genesToLoad, genesAlreadyLoaded } = changed + ? { genesToLoad: genes, genesAlreadyLoaded: [] } + : findLoadedGenesInDownsampled(downsampled.matrix, genes); + + // Nothing to do — all genes already in downsampled matrix with same settings + if (genesToLoad.length === 0) return null; + + dispatch({ type: HEATMAP_EXPRESSION_LOADING }); + + // For append requests (settings unchanged), always send the stored cell IDs so the worker + // returns expression for exactly the same cells already in the matrix. Without this, + // bucketed requests would re-sample randomly and produce expression for different cells. + const downsampleSettings = !changed + ? { cellIds: downsampled.cellIds } + : isBucketed + ? { selectedCellSet, groupedTracks } + : { cellIds: cellIdsToRequest }; + + const body = { + name: 'GeneExpression', + genes: genesToLoad, + downsampleSettings, + }; + + const timeout = getTimeoutForWorkerTask(state, 'GeneExpression'); + + try { + const { + orderedGeneNames, + rawExpression: rawExpressionJson, + stats, + cellIds: workerCellIds, + } = await fetchWork(experimentId, body, getState, dispatch, { timeout }); + + const rawExpression = SparseMatrix.fromJSON(rawExpressionJson); + + dispatch({ + type: HEATMAP_EXPRESSION_LOADED, + payload: { + newGenes: { orderedGeneNames, rawExpression, stats }, + cellIds: changed ? (workerCellIds ?? cellIdsToRequest) : downsampled.cellIds, + downsampleType, + lastFetchSettings: newSettings, + isAppend: !changed, + genesAlreadyLoaded, + }, + }); + } catch (error) { + dispatch({ + type: HEATMAP_EXPRESSION_ERROR, + payload: { error }, + }); + } + + return null; +}; + +export { LARGE_DATASET_THRESHOLD }; +export default loadHeatmapExpression; diff --git a/src/redux/reducers/componentConfig/initialState.js b/src/redux/reducers/componentConfig/initialState.js index 65f10e2d8d..7713f24d07 100644 --- a/src/redux/reducers/componentConfig/initialState.js +++ b/src/redux/reducers/componentConfig/initialState.js @@ -203,7 +203,7 @@ const heatmapInitialConfig = { selectedPoints: 'All', labelColour: 'transparent', selectedTracks: ['louvain'], - groupedTracks: ['louvain', 'sample'], + groupedTracks: ['louvain'], expressionValue: 'raw', truncatedValues: true, geneLabelSize: 10, @@ -639,7 +639,7 @@ const embeddingPreviewNumOfUmisInitialConfig = { const interactiveHeatmapInitialConfig = { selectedCellSet: 'louvain', selectedTracks: ['louvain'], - groupedTracks: ['louvain', 'sample'], + groupedTracks: ['louvain'], selectedGenes: [], expressionValue: 'raw', legendIsVisible: true, diff --git a/src/redux/reducers/genes/getInitialState.js b/src/redux/reducers/genes/getInitialState.js index 0a74351f1e..04648c9116 100644 --- a/src/redux/reducers/genes/getInitialState.js +++ b/src/redux/reducers/genes/getInitialState.js @@ -20,6 +20,14 @@ const getInitialState = () => ({ ETag: null, matrix: new ExpressionMatrix(), }, + downsampled: { + loading: false, + error: false, + cellIds: [], + downsampleType: null, // 'bucketed' | 'precomputed' | null + lastFetchSettings: null, // { downsampleType, selectedCellSet, groupedTracks, cellIdsKey } + matrix: new ExpressionMatrix(), + }, }, selected: [], focused: undefined, diff --git a/src/redux/reducers/genes/heatmapExpressionError.js b/src/redux/reducers/genes/heatmapExpressionError.js new file mode 100644 index 0000000000..aa424b3da7 --- /dev/null +++ b/src/redux/reducers/genes/heatmapExpressionError.js @@ -0,0 +1,17 @@ +const heatmapExpressionError = (state, action) => { + const { error } = action.payload; + + return { + ...state, + expression: { + ...state.expression, + downsampled: { + ...state.expression.downsampled, + loading: false, + error, + }, + }, + }; +}; + +export default heatmapExpressionError; diff --git a/src/redux/reducers/genes/heatmapExpressionLoaded.js b/src/redux/reducers/genes/heatmapExpressionLoaded.js new file mode 100644 index 0000000000..03f2d16d66 --- /dev/null +++ b/src/redux/reducers/genes/heatmapExpressionLoaded.js @@ -0,0 +1,48 @@ +const heatmapExpressionLoaded = (state, action) => { + const { + newGenes, + cellIds, + downsampleType, + lastFetchSettings, + isAppend, + } = action.payload; + + if (newGenes) { + const { orderedGeneNames, rawExpression, stats } = newGenes; + + if (isAppend) { + // Only new genes added; matrix column order (cells) is unchanged + state.expression.downsampled.matrix.pushGeneExpression( + orderedGeneNames, + rawExpression, + stats, + ); + } else { + // Full overwrite: reset matrix with new cell set + state.expression.downsampled.matrix.setGeneExpression( + orderedGeneNames, + rawExpression, + stats, + ); + } + } + + return { + ...state, + expression: { + ...state.expression, + downsampled: { + ...state.expression.downsampled, + loading: false, + error: false, + ...(!isAppend && newGenes && { + cellIds, + downsampleType, + lastFetchSettings, + }), + }, + }, + }; +}; + +export default heatmapExpressionLoaded; diff --git a/src/redux/reducers/genes/heatmapExpressionLoading.js b/src/redux/reducers/genes/heatmapExpressionLoading.js new file mode 100644 index 0000000000..44ef7d475c --- /dev/null +++ b/src/redux/reducers/genes/heatmapExpressionLoading.js @@ -0,0 +1,13 @@ +const heatmapExpressionLoading = (state) => ({ + ...state, + expression: { + ...state.expression, + downsampled: { + ...state.expression.downsampled, + loading: true, + error: false, + }, + }, +}); + +export default heatmapExpressionLoading; diff --git a/src/redux/reducers/genes/index.js b/src/redux/reducers/genes/index.js index 12209bc071..d2056d385f 100644 --- a/src/redux/reducers/genes/index.js +++ b/src/redux/reducers/genes/index.js @@ -4,6 +4,7 @@ import { GENES_SELECT, GENES_DESELECT, GENES_EXPRESSION_LOADING, GENES_EXPRESSION_LOADED, GENES_EXPRESSION_ERROR, MARKER_GENES_LOADING, MARKER_GENES_LOADED, MARKER_GENES_ERROR, + HEATMAP_EXPRESSION_LOADING, HEATMAP_EXPRESSION_LOADED, HEATMAP_EXPRESSION_ERROR, } from 'redux/actionTypes/genes'; import { EXPERIMENT_SETTINGS_QC_START } from 'redux/actionTypes/experimentSettings'; @@ -20,6 +21,10 @@ import markerGenesLoading from 'redux/reducers/genes/markerGenesLoading'; import markerGenesError from 'redux/reducers/genes/markerGenesError'; import markerGenesLoaded from 'redux/reducers/genes/markerGenesLoaded'; +import heatmapExpressionLoading from 'redux/reducers/genes/heatmapExpressionLoading'; +import heatmapExpressionLoaded from 'redux/reducers/genes/heatmapExpressionLoaded'; +import heatmapExpressionError from 'redux/reducers/genes/heatmapExpressionError'; + import genesSelect from 'redux/reducers/genes/genesSelect'; import genesDeselect from 'redux/reducers/genes/genesDeselect'; @@ -61,6 +66,15 @@ const genesReducer = (state = getInitialState(), action) => { case MARKER_GENES_ERROR: { return markerGenesError(state, action); } + case HEATMAP_EXPRESSION_LOADING: { + return heatmapExpressionLoading(state, action); + } + case HEATMAP_EXPRESSION_LOADED: { + return heatmapExpressionLoaded(state, action); + } + case HEATMAP_EXPRESSION_ERROR: { + return heatmapExpressionError(state, action); + } default: { return state; } diff --git a/src/utils/calculateNMarkerGenes.js b/src/utils/calculateNMarkerGenes.js deleted file mode 100644 index 2b3f4bd841..0000000000 --- a/src/utils/calculateNMarkerGenes.js +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Calculates the number of marker genes to display based on dataset size and cluster count. - * Logic: - * - Default: 5 markers (max) - * - If > 500K cells: max = 30 / clusterCount (min 1) - * - If > 250K cells: max = 60 / clusterCount (min 1) - * - Otherwise: 5 markers - * - * @param {number} totalCells - Total number of cells in the dataset - * @param {number} clusterCount - Number of clusters - * @returns {number} Number of marker genes to display (min 1, max 5) - */ -const calculateNMarkerGenes = (totalCells, clusterCount) => { - const DEFAULT_MARKERS = 5; - const MIN_MARKERS = 1; - - // If we don't have valid data, use default - if (!totalCells || !clusterCount) { - return DEFAULT_MARKERS; - } - - if (totalCells > 500000) { - // (clusters * markers) < 30 - return Math.max(MIN_MARKERS, Math.min(DEFAULT_MARKERS, Math.floor(30 / clusterCount))); - } - - if (totalCells > 250000) { - // (clusters * markers) < 60 - return Math.max(MIN_MARKERS, Math.min(DEFAULT_MARKERS, Math.floor(60 / clusterCount))); - } - - // Default for smaller datasets - return DEFAULT_MARKERS; -}; - -export default calculateNMarkerGenes; diff --git a/src/utils/plotSpecs/generateEmbeddingCategoricalSpec.js b/src/utils/plotSpecs/generateEmbeddingCategoricalSpec.js index 8de496dd96..5788301c43 100644 --- a/src/utils/plotSpecs/generateEmbeddingCategoricalSpec.js +++ b/src/utils/plotSpecs/generateEmbeddingCategoricalSpec.js @@ -1,6 +1,5 @@ /* eslint-disable no-param-reassign */ import _ from 'lodash'; -import { getAllCells, getSampleCells } from 'utils/cellSets'; const paddingSize = 5; @@ -186,13 +185,6 @@ const generateSpec = (config, method, plotData, cellSetLegendsData) => { { name: 'values', values: plotData, - // Vega internally modifies objects during data transforms. If the plot data is frozen, - // Vega is not able to carry out the transform and will throw an error. - // https://github.com/vega/vega/issues/2453#issuecomment-604516777 - format: { - type: 'json', - copy: true, - }, }, { name: 'labels', @@ -303,87 +295,65 @@ const generateSpec = (config, method, plotData, cellSetLegendsData) => { }; const filterCells = (cellSets, sampleKey, groupBy) => { - let filteredCells = []; - - // Get all the filtered cells - if (sampleKey === 'All') { - filteredCells = getAllCells(cellSets, groupBy); - } else { - filteredCells = getSampleCells(cellSets, sampleKey); - } - - // Get the cell set names - const clusterEnteries = cellSets.hierarchy - .find( - (rootNode) => rootNode.key === groupBy, - )?.children || []; - - const cellSetKeys = clusterEnteries.map(({ key }) => key); - - const colorToCellIdsMap = cellSetKeys.reduce((acc, key) => { - acc.push({ - cellIds: cellSets.properties[key].cellIds, - key, - name: cellSets.properties[key].name, - color: cellSets.properties[key].color, - }); - - return acc; - }, []); - - let cellSetLegendsData = []; + const clusterEntries = cellSets.hierarchy + .find((rootNode) => rootNode.key === groupBy)?.children || []; + + const cellSetKeys = clusterEntries.map(({ key }) => key); + + // Build a reverse lookup map: cellId -> cluster info in O(n_cells). + // This replaces the previous O(n_cells × n_clusters) linear .find() per cell. + const cellIdToCluster = new Map(); + cellSetKeys.forEach((key) => { + const { name, color, cellIds } = cellSets.properties[key]; + cellIds.forEach((cellId) => cellIdToCluster.set(cellId, { key, name, color })); + }); + + // For a specific sample, keep the existing cellIds Set for O(1) membership checks. + const sampleCellIds = sampleKey === 'All' + ? null + : (cellSets.properties[sampleKey]?.cellIds ?? new Set()); + + const filteredCells = {}; + const cellSetLegendsData = []; const addedCellSetKeys = new Set(); - filteredCells = filteredCells.reduce((acc, cell) => { - if (!cell) return acc; - - const inCellSet = colorToCellIdsMap.find((map) => map.cellIds.has(cell.cellId)); - - // If cell is not in the cell set, then return - if (!inCellSet) return acc; - - const { key, name, color } = inCellSet; + cellIdToCluster.forEach(({ key, name, color }, cellId) => { + if (sampleCellIds && !sampleCellIds.has(cellId)) return; - if (!addedCellSetKeys.has(key)) { - addedCellSetKeys.add(key); - cellSetLegendsData.push({ key, name, color }); - } - - acc[cell.cellId] = { - ...cell, + filteredCells[cellId] = { + cellId, cellSetKey: key, cellSetName: name, color, }; - return acc; - }, {}); + if (!addedCellSetKeys.has(key)) { + addedCellSetKeys.add(key); + cellSetLegendsData.push({ key, name, color }); + } + }); // Sort legends to show them in the order that cellSetKeys are stored - cellSetLegendsData = _.sortBy( - cellSetLegendsData, - ({ key }) => _.indexOf(cellSetKeys, key), - ); - - return { filteredCells, cellSetLegendsData }; + return { + filteredCells, + cellSetLegendsData: _.sortBy(cellSetLegendsData, ({ key }) => _.indexOf(cellSetKeys, key)), + }; }; // Generate dynamic data from redux store const generateData = (cellSets, sampleKey, groupBy, embeddingData) => { const { filteredCells, cellSetLegendsData } = filterCells(cellSets, sampleKey, groupBy); - const plotData = embeddingData - .map((coordinates, cellId) => ({ cellId, coordinates })) - .filter(({ coordinates }) => coordinates !== undefined) - .map((data) => { - const { cellId, coordinates } = data; - - return { - ...filteredCells[cellId], - x: coordinates[0], - y: coordinates[1], - }; + // Single-pass forEach avoids creating intermediate arrays / objects from chained .map()/.filter() + const plotData = []; + embeddingData.forEach((coordinates, cellId) => { + if (coordinates === undefined || !filteredCells[cellId]) return; + plotData.push({ + ...filteredCells[cellId], + x: coordinates[0], + y: coordinates[1], }); + }); return { plotData, cellSetLegendsData }; }; diff --git a/src/utils/plotSpecs/generateEmbeddingContinuousSpec.js b/src/utils/plotSpecs/generateEmbeddingContinuousSpec.js index 96f4d39c1b..822ef71249 100644 --- a/src/utils/plotSpecs/generateEmbeddingContinuousSpec.js +++ b/src/utils/plotSpecs/generateEmbeddingContinuousSpec.js @@ -1,7 +1,5 @@ /* eslint-disable no-param-reassign */ -import { getAllCells, getSampleCells } from 'utils/cellSets'; - const generateSpec = (config, method, plotData) => { const xScaleDomain = config.axesRanges.xAxisAuto ? { data: 'plotData', field: 'x' } @@ -181,15 +179,10 @@ const generateSpec = (config, method, plotData) => { }; const filterCells = (cellSets, selectedSample) => { - let filteredCells = []; - - if (selectedSample === 'All') { - filteredCells = getAllCells(cellSets); - } else { - filteredCells = getSampleCells(cellSets, selectedSample); - } - - return new Set(filteredCells.map((cell) => cell.cellId)); + // For 'All', return null to indicate no sample filter — avoids building a 1M-entry Set + if (selectedSample === 'All') return null; + // Reuse the existing cellIds Set from properties rather than constructing a new one + return cellSets.properties[selectedSample]?.cellIds ?? new Set(); }; const generateData = ( @@ -198,21 +191,19 @@ const generateData = ( plotData, embeddingData, ) => { - const filteredCells = filterCells(cellSets, selectedSample, embeddingData); - - const cells = embeddingData - .map((coordinates, cellId) => ({ cellId, coordinates })) - .filter(({ coordinates }) => coordinates !== undefined) - .filter(({ cellId }) => filteredCells.has(cellId)) - .map((data) => { - const { cellId, coordinates } = data; + const filteredCells = filterCells(cellSets, selectedSample); - return { - x: coordinates[0], - y: coordinates[1], - value: plotData[cellId], - }; + // Single-pass forEach avoids chained .map()/.filter() intermediate arrays + const cells = []; + embeddingData.forEach((coordinates, cellId) => { + if (coordinates === undefined) return; + if (filteredCells !== null && !filteredCells.has(cellId)) return; + cells.push({ + x: coordinates[0], + y: coordinates[1], + value: plotData[cellId], }); + }); return cells; }; diff --git a/src/utils/work/getHeatmapCellOrder.js b/src/utils/work/getHeatmapCellOrder.js index 821dd5dac2..57452692b5 100644 --- a/src/utils/work/getHeatmapCellOrder.js +++ b/src/utils/work/getHeatmapCellOrder.js @@ -1,238 +1,288 @@ import seedrandom from 'seedrandom'; import lruMemoize from 'lru-memoize'; -/** - * Downsamples cell order for marker heatmap based on: - * - Selected cell set - * - Grouped tracks (cartesian product buckets) - * - Hidden cell sets (to exclude) - * - Maximum cells threshold - * - * This logic was previously in the Python worker; now it's client-side. - * Mirrors the logic in worker/python/src/worker/helpers/get_heatmap_cell_order.py - * - * @param {string} selectedCellSet - Key of the primary cell set (e.g., "louvain") - * @param {string[]} groupedTracks - Array of cell set keys for cartesian product - * @param {Set|string[]} hiddenCellSets - Cell set keys to exclude - * @param {object} cellSets - Cell sets structure with hierarchy and properties - * @param {number} maxCells - Maximum cells to return after downsampling (default 1000) - * @returns {number[]} Array of cell IDs to display, downsampled proportionally - */ +// ─── Module-level helpers (not exported) ──────────────────────────────────── + +const getCellClassIds = (key, hierarchy, properties) => { + const cellClassNode = hierarchy.find((node) => node.key === key); + if (!cellClassNode) return new Set(); + + const cellIds = new Set(); + cellClassNode.children.forEach((child) => { + const childCellIds = properties[child.key]?.cellIds || new Set(); + childCellIds.forEach((id) => cellIds.add(id)); + }); + return cellIds; +}; -const getHeatmapCellOrder = ( - selectedCellSet, - groupedTracks, - hiddenCellSets, - cellSets, - maxCells = 1000, -) => { - if ( - !cellSets - || !cellSets.hierarchy - || !cellSets.properties - || !groupedTracks - || !selectedCellSet - ) { - return []; - } +const getCells = (key, isRootNode, filteredCellIds, hierarchy, properties) => { + const unfilteredCellIds = isRootNode + ? getCellClassIds(key, hierarchy, properties) + : (properties[key]?.cellIds || new Set()); - const { hierarchy, properties } = cellSets; + const intersection = new Set(); + unfilteredCellIds.forEach((id) => { + if (filteredCellIds.has(id)) intersection.add(id); + }); + return intersection; +}; - // Normalize hiddenCellSets before using it (could be null, undefined, Set, or array) - const normalizedHiddenCellSets = hiddenCellSets instanceof Set - ? Array.from(hiddenCellSets) - : (hiddenCellSets || []); +const getEnabledCellIds = ( + selectedCellSet, normalizedHiddenCellSets, filteredCellIds, hierarchy, properties, +) => { + const cellIds = getCells(selectedCellSet, true, filteredCellIds, hierarchy, properties); - // Create a seed from the function inputs to ensure deterministic downsampling - const seedString = `${selectedCellSet}|${groupedTracks.join(',')}|${normalizedHiddenCellSets.join(',')}`; - const random = seedrandom(seedString); + normalizedHiddenCellSets.forEach((hiddenKey) => { + const hiddenCellIds = getCells(hiddenKey, false, filteredCellIds, hierarchy, properties); + hiddenCellIds.forEach((id) => cellIds.delete(id)); + }); - // Helper to get all cell IDs in a cell class (root node) - const getCellClassIds = (key) => { - const cellClassNode = hierarchy.find((node) => node.key === key); - if (!cellClassNode) return new Set(); + return cellIds; +}; - const cellIds = new Set(); - cellClassNode.children.forEach((child) => { - const childCellIds = properties[child.key]?.cellIds || new Set(); - childCellIds.forEach((id) => cellIds.add(id)); - }); - return cellIds; - }; +const getIntersections = (bucket, cellClass, hierarchy, properties) => { + const cellClassNode = hierarchy.find((node) => node.key === cellClass); + if (!cellClassNode) return []; - // Start with all cells from louvain (as in the Python implementation) - const filteredCellIds = getCellClassIds('louvain'); + const intersections = []; - // Helper to get cells, intersecting with filteredCellIds - const getCells = (key, isRootNode = false) => { - let unfilteredCellIds; + cellClassNode.children.forEach((child) => { + const childCellIds = properties[child.key]?.cellIds || new Set(); + const intersection = new Set(); - if (isRootNode) { - unfilteredCellIds = getCellClassIds(key); + if (childCellIds.size < bucket.size) { + childCellIds.forEach((id) => { + if (bucket.has(id)) intersection.add(id); + }); } else { - unfilteredCellIds = properties[key]?.cellIds || new Set(); + bucket.forEach((id) => { + if (childCellIds.has(id)) intersection.add(id); + }); } - // Intersect with filtered_cell_ids (cells in louvain) - // Optimize: iterate through smaller set instead of spreading larger set - const intersection = new Set(); - unfilteredCellIds.forEach((id) => { - if (filteredCellIds.has(id)) { - intersection.add(id); - } - }); + if (intersection.size > 0) intersections.push(intersection); + }); + + return intersections; +}; - return intersection; - }; +const cartesianProductIntersection = (buckets, cellClass, hierarchy, properties) => { + const newBuckets = []; - // Get all enabled (non-hidden) cells - const getAllEnabledCellIds = () => { - // Get cells from the selected cell set - const cellIds = getCells(selectedCellSet, true); + buckets.forEach((bucket) => { + const intersections = getIntersections(bucket, cellClass, hierarchy, properties); - // Remove hidden cells - avoid array conversion - normalizedHiddenCellSets.forEach((hiddenKey) => { - const hiddenCellIds = getCells(hiddenKey); - hiddenCellIds.forEach((id) => { - cellIds.delete(id); - }); + const leftoverCells = new Set(bucket); + intersections.forEach((intersection) => { + intersection.forEach((id) => leftoverCells.delete(id)); }); - return cellIds; - }; + intersections.forEach((intersection) => newBuckets.push(intersection)); + if (leftoverCells.size > 0) newBuckets.push(leftoverCells); + }); - // Get intersections of a bucket with a cell class - const getIntersections = (bucket, cellClass) => { - const cellClassNode = hierarchy.find((node) => node.key === cellClass); - if (!cellClassNode) return []; + return newBuckets; +}; - const intersections = []; +const splitByCartesianIntersections = (enabledCellIds, groupedTracks, hierarchy, properties) => { + let buckets = [enabledCellIds]; - // For each child in the cell class - cellClassNode.children.forEach((child) => { - const childCellIds = properties[child.key]?.cellIds || new Set(); - const intersection = new Set(); + groupedTracks.forEach((cellClass) => { + buckets = cartesianProductIntersection(buckets, cellClass, hierarchy, properties); + }); - // Only iterate through the smaller set to build intersection - if (childCellIds.size < bucket.size) { - childCellIds.forEach((id) => { - if (bucket.has(id)) { - intersection.add(id); - } - }); - } else { - bucket.forEach((id) => { - if (childCellIds.has(id)) { - intersection.add(id); - } - }); - } + let totalSize = 0; + buckets.forEach((bucket) => { totalSize += bucket.size; }); - if (intersection.size > 0) { - intersections.push(intersection); - } - }); + return { buckets, totalSize }; +}; - return intersections; - }; +const sampleFromBucket = (bucketArray, sampleSize, random) => { + const sample = []; + const bucketCopy = [...bucketArray]; - // Perform cartesian product intersection for a single track - const cartesianProductIntersection = (buckets, cellClass) => { - const newBuckets = []; + for (let i = 0; i < sampleSize; i += 1) { + const randomIndex = Math.floor(random() * bucketCopy.length); + sample.push(bucketCopy[randomIndex]); + bucketCopy.splice(randomIndex, 1); + } - buckets.forEach((bucket) => { - const intersections = getIntersections(bucket, cellClass); + return sample; +}; - // Calculate leftover cells (not in any intersection) - avoid array conversion - const leftoverCells = new Set(bucket); - intersections.forEach((intersection) => { - // Remove intersection cells from leftover without converting to array - intersection.forEach((id) => { - leftoverCells.delete(id); - }); - }); +const downsampleBuckets = (buckets, totalSize, maxCells, random) => { + const downsampledCellIds = []; + const finalSampleSize = Math.min(totalSize, maxCells); - // Add all intersections - intersections.forEach((intersection) => { - newBuckets.push(intersection); - }); + buckets.forEach((bucket) => { + const sampleSize = Math.floor((bucket.size / totalSize) * finalSampleSize); + if (sampleSize > 0) { + downsampledCellIds.push(...sampleFromBucket(Array.from(bucket), sampleSize, random)); + } + }); - // Add leftover cells as their own bucket - if (leftoverCells.size > 0) { - newBuckets.push(leftoverCells); - } - }); + return downsampledCellIds; +}; + +// ─── Exported functions ────────────────────────────────────────────────────── + +/** + * Returns cartesian-product buckets without final downsampling. + * + * @param {string} selectedCellSet + * @param {string[]} groupedTracks + * @param {Set|string[]} hiddenCellSets + * @param {object} cellSets - Redux cellSets { hierarchy, properties } + * @returns {{ buckets: Set[], totalSize: number }} + */ +const getBuckets = (selectedCellSet, groupedTracks, hiddenCellSets, cellSets) => { + if ( + !cellSets?.hierarchy + || !cellSets?.properties + || !groupedTracks + || !selectedCellSet + ) { + return { buckets: [], totalSize: 0 }; + } - return newBuckets; - }; + const { hierarchy, properties } = cellSets; - // Split into cartesian product buckets - const splitByCartesianIntersections = (enabledCellIds) => { - let buckets = [enabledCellIds]; + const normalizedHiddenCellSets = hiddenCellSets instanceof Set + ? Array.from(hiddenCellSets) + : (hiddenCellSets || []); - // For each grouped track, split buckets by intersection - groupedTracks.forEach((cellClass) => { - buckets = cartesianProductIntersection(buckets, cellClass); - }); + const filteredCellIds = getCellClassIds('louvain', hierarchy, properties); - // Calculate total size across all buckets - let totalSize = 0; - buckets.forEach((bucket) => { - totalSize += bucket.size; - }); + const enabledCellIds = getEnabledCellIds( + selectedCellSet, normalizedHiddenCellSets, filteredCellIds, hierarchy, properties, + ); - return { buckets, totalSize }; - }; + if (groupedTracks.length === 0 || enabledCellIds.size === 0) { + return { buckets: [], totalSize: 0 }; + } - // Downsample buckets proportionally - const downsample = (buckets, totalSize) => { - const downsampledCellIds = []; - const finalSampleSize = Math.min(totalSize, maxCells); + return splitByCartesianIntersections(enabledCellIds, groupedTracks, hierarchy, properties); +}; - buckets.forEach((bucket) => { - const sampleSize = Math.floor( - (bucket.size / totalSize) * finalSampleSize, - ); +/** + * Bucketed downsampling second pass: compute display cell IDs after the worker has + * returned expression data for up to maxCells cells per cartesian-product bucket. + * + * Uses full-bucket proportions (no hidden sets) to determine per-bucket sample counts, + * but restricts actual sampling to cells that are: + * (a) not in hiddenCellSets, and + * (b) present in workerCellIds (have expression data from the worker). + * + * @param {string} selectedCellSet + * @param {string[]} groupedTracks + * @param {Set|string[]} hiddenCellSets + * @param {object} cellSets + * @param {number[]} workerCellIds - ordered cell IDs returned by the worker + * @param {number} [maxCells=5000] - maximum number of cells to return after downsampling + * @returns {number[]} display cell IDs + */ +const computeBucketedDisplayCellIds = ( + selectedCellSet, + groupedTracks, + hiddenCellSets, + cellSets, + workerCellIds, + maxCells = 5000, +) => { + if (!workerCellIds?.length || !cellSets?.hierarchy || !cellSets?.properties) { + return []; + } - if (sampleSize > 0) { - // Deterministic sample from bucket using seeded random - const bucketArray = Array.from(bucket); - const sample = []; - const bucketCopy = [...bucketArray]; + const normalizedHiddenCellSets = hiddenCellSets instanceof Set + ? Array.from(hiddenCellSets) + : (hiddenCellSets || []); - for (let i = 0; i < sampleSize; i += 1) { - const randomIndex = Math.floor(random() * bucketCopy.length); - sample.push(bucketCopy[randomIndex]); - bucketCopy.splice(randomIndex, 1); - } + // Full buckets (no hidden) to get cartesian-product bucket membership for each cell + const { buckets: fullBuckets } = getBuckets( + selectedCellSet, groupedTracks, [], cellSets, + ); - downsampledCellIds.push(...sample); - } - }); + if (fullBuckets.length === 0) return []; - return downsampledCellIds; - }; + const { hierarchy, properties } = cellSets; + const filteredCellIds = getCellClassIds('louvain', hierarchy, properties); + + // Build hidden cell IDs set (intersected with louvain) + const hiddenCellIdsSet = new Set(); + normalizedHiddenCellSets.forEach((hiddenKey) => { + const hiddenIds = properties[hiddenKey]?.cellIds || new Set(); + hiddenIds.forEach((id) => { + if (filteredCellIds.has(id)) hiddenCellIdsSet.add(id); + }); + }); - const enabledCellIds = getAllEnabledCellIds(); + const workerCellIdsSet = new Set(workerCellIds); - if (groupedTracks.length === 0 || enabledCellIds.size === 0) { - return []; - } + // Deterministic seed encodes inputs that affect which cells are returned by the worker + const seedString = `bucketed|${selectedCellSet}|${groupedTracks.join(',')}|${normalizedHiddenCellSets.join(',')}|${workerCellIds.length}`; + const random = seedrandom(seedString); - const { buckets, totalSize } = splitByCartesianIntersections(enabledCellIds); - const result = downsample(buckets, totalSize); + // For each bucket: compute effective size (full size minus hidden cells) and eligible pool + // (worker cells that are not hidden). Proportions use effective sizes so they reflect + // actual cluster sizes in the dataset, not the capped worker sample counts. + const bucketData = fullBuckets.map((bucket) => { + let hiddenCount = 0; + bucket.forEach((id) => { if (hiddenCellIdsSet.has(id)) hiddenCount += 1; }); + const effectiveSize = bucket.size - hiddenCount; + const eligible = Array.from(bucket).filter( + (id) => !hiddenCellIdsSet.has(id) && workerCellIdsSet.has(id), + ); + return { effectiveSize, eligible }; + }); + + const totalEffectiveSize = bucketData.reduce((sum, { effectiveSize }) => sum + effectiveSize, 0); + if (totalEffectiveSize === 0) return []; + + // Pass 1: guarantee each non-empty bucket a minimum, capped to what's eligible. + const minCellsBerBucket = Math.max( + 1, Math.round(Math.min(totalEffectiveSize, maxCells) * 0.01), + ); + + const withGuarantee = bucketData.map(({ effectiveSize, eligible }) => ({ + effectiveSize, + eligible, + guaranteed: Math.min(minCellsBerBucket, eligible.length), + })); + + const totalGuaranteed = withGuarantee.reduce((sum, { guaranteed }) => sum + guaranteed, 0); + const remainingQuota = Math.max(0, maxCells - totalGuaranteed); + + // Pass 2: distribute remaining quota proportionally by effectiveSize among buckets + // that have eligible cells beyond their guarantee. + const totalExcessEffectiveSize = withGuarantee.reduce( + (sum, { effectiveSize, eligible, guaranteed }) => ( + eligible.length > guaranteed ? sum + effectiveSize : sum + ), + 0, + ); + + const displayCellIds = []; + + withGuarantee.forEach(({ effectiveSize, eligible, guaranteed }) => { + if (eligible.length === 0) return; + let extra = 0; + if (eligible.length > guaranteed && totalExcessEffectiveSize > 0) { + extra = Math.floor((effectiveSize / totalExcessEffectiveSize) * remainingQuota); + } + const sampleCount = Math.min(guaranteed + extra, eligible.length); + if (sampleCount === 0) return; + displayCellIds.push(...sampleFromBucket(eligible, sampleCount, random)); + }); - return result; + return displayCellIds; }; /** - * Compute which cell sets should be hidden based on user selections - * Mirrors the logic in updateDownsampledCellOrder thunk + * Compute which cell sets should be hidden based on user selections. */ const computeHiddenCellSets = (selectedPoints, cellSets) => { let hiddenCellSets = Array.from(cellSets.hidden || []); - // Compute which cell sets should be hidden based on selectedPoints if (selectedPoints && selectedPoints !== 'All') { const parts = selectedPoints.split('/'); if (parts.length === 2) { @@ -248,10 +298,7 @@ const computeHiddenCellSets = (selectedPoints, cellSets) => { hiddenCellSets.push(child.key); } } else { - // Include the selected cell set even if it was manually hidden - hiddenCellSets = hiddenCellSets.filter( - (key) => key !== child.key, - ); + hiddenCellSets = hiddenCellSets.filter((key) => key !== child.key); } }); } @@ -261,8 +308,54 @@ const computeHiddenCellSets = (selectedPoints, cellSets) => { return hiddenCellSets; }; -// Memoize getHeatmapCellOrder to avoid recomputation with same inputs +/** + * Downsamples cell order for marker heatmap. + * + * @param {string} selectedCellSet - Key of the primary cell set (e.g., "louvain") + * @param {string[]} groupedTracks - Array of cell set keys for cartesian product + * @param {Set|string[]} hiddenCellSets - Cell set keys to exclude + * @param {object} cellSets - Cell sets structure with hierarchy and properties + * @param {number} maxCells - Maximum cells to return after downsampling (default 1000) + * @returns {number[]} Array of cell IDs to display, downsampled proportionally + */ +const getHeatmapCellOrder = ( + selectedCellSet, + groupedTracks, + hiddenCellSets, + cellSets, + maxCells = 1000, +) => { + if ( + !cellSets + || !cellSets.hierarchy + || !cellSets.properties + || !groupedTracks + || !selectedCellSet + ) { + return []; + } + + const normalizedHiddenCellSets = hiddenCellSets instanceof Set + ? Array.from(hiddenCellSets) + : (hiddenCellSets || []); + + const seedString = [ + selectedCellSet, + groupedTracks.join(','), + normalizedHiddenCellSets.join(','), + ].join('|'); + const random = seedrandom(seedString); + + const { buckets, totalSize } = getBuckets( + selectedCellSet, groupedTracks, hiddenCellSets, cellSets, + ); + + if (buckets.length === 0 || totalSize === 0) return []; + + return downsampleBuckets(buckets, totalSize, maxCells, random); +}; + const memoizedGetHeatmapCellOrder = lruMemoize(10)(getHeatmapCellOrder); -export { computeHiddenCellSets }; +export { computeHiddenCellSets, getBuckets, computeBucketedDisplayCellIds }; export default memoizedGetHeatmapCellOrder;