From d285c6642ddd370f1e7a1e4c1557820bd42f1d3e Mon Sep 17 00:00:00 2001 From: Xun Li Date: Thu, 19 Oct 2023 14:20:32 -0700 Subject: [PATCH] add gpu filter --- .../src/geojson-layer/filter-arrow-layer.ts | 50 +++++++++++++++++++ .../src/geojson-layer/filter-shader-module.ts | 39 +++++++++++++++ src/deckgl-layers/src/index.ts | 1 + src/layers/src/arrow-layer/arrow-layer.ts | 30 +++++++++-- src/utils/src/arrow-utils.ts | 2 +- 5 files changed, 118 insertions(+), 4 deletions(-) create mode 100644 src/deckgl-layers/src/geojson-layer/filter-arrow-layer.ts create mode 100644 src/deckgl-layers/src/geojson-layer/filter-shader-module.ts diff --git a/src/deckgl-layers/src/geojson-layer/filter-arrow-layer.ts b/src/deckgl-layers/src/geojson-layer/filter-arrow-layer.ts new file mode 100644 index 0000000000..18cb80be96 --- /dev/null +++ b/src/deckgl-layers/src/geojson-layer/filter-arrow-layer.ts @@ -0,0 +1,50 @@ +import {Layer, LayerExtension} from '@deck.gl/core'; +import {LayerContext} from '@deck.gl/core/lib/layer'; +import GL from '@luma.gl/constants'; + +import shaderModule from './filter-shader-module'; + +const defaultProps = { + getFiltered: {type: 'accessor', value: 1} +}; + +export type FilterArrowExtensionProps = { + getFiltered?: () => number; +}; + +// Write an extension to filter arrow layer: +// an instanced attribute 'instanceFiltered' is added to the layer to indicate whether the feature has been Filtered +// the shader module is modified to discard the feature if instanceFiltered is 0 +// the accessor getFiltered is used to get the value of instanceFiltered based on filteredIndex in Arrowlayer +export default class FilterArrowExtension extends LayerExtension { + static defaultProps = defaultProps; + static extensionName = 'FilterArrowExtension'; + + getShaders(extension: any) { + return { + modules: [shaderModule], + defines: {} + }; + } + + initializeState(this: Layer, context: LayerContext, extension: this) { + const attributeManager = this.getAttributeManager(); + if (attributeManager) { + attributeManager.add({ + filtered: { + size: 1, + type: GL.FLOAT, + accessor: 'getFiltered', + shaderAttributes: { + filtered: { + divisor: 0 + }, + instanceFiltered: { + divisor: 1 + } + } + } + }); + } + } +} diff --git a/src/deckgl-layers/src/geojson-layer/filter-shader-module.ts b/src/deckgl-layers/src/geojson-layer/filter-shader-module.ts new file mode 100644 index 0000000000..5e5755eda1 --- /dev/null +++ b/src/deckgl-layers/src/geojson-layer/filter-shader-module.ts @@ -0,0 +1,39 @@ +import {project} from '@deck.gl/core'; + +const vs = ` + #ifdef NON_INSTANCED_MODEL + #define FILTER_ARROW_ATTRIB filtered + #else + #define FILTER_ARROW_ATTRIB instanceFiltered + #endif + attribute float FILTER_ARROW_ATTRIB; +`; + +const fs = ``; + +const inject = { + 'vs:#decl': ` + varying float is_filtered; + `, + 'vs:#main-end': ` + is_filtered = FILTER_ARROW_ATTRIB; + `, + 'fs:#decl': ` + varying float is_filtered; + `, + 'fs:DECKGL_FILTER_COLOR': ` + // abandon the fragments if it is not filtered + if (is_filtered == 0.) { + discard; + } + ` +}; + +export default { + name: 'filter-arrow', + dependencies: [project], + vs: vs, + fs: fs, + inject: inject, + getUniforms: () => {} +} diff --git a/src/deckgl-layers/src/index.ts b/src/deckgl-layers/src/index.ts index 459c283ede..bcec4e74eb 100644 --- a/src/deckgl-layers/src/index.ts +++ b/src/deckgl-layers/src/index.ts @@ -9,6 +9,7 @@ export {default as EnhancedGridLayer} from './grid-layer/enhanced-cpu-grid-layer export {default as EnhancedHexagonLayer} from './hexagon-layer/enhanced-hexagon-layer'; export {default as EnhancedLineLayer} from './line-layer/line-layer'; export {default as SvgIconLayer} from './svg-icon-layer/svg-icon-layer'; +export {default as FilterArrowExtension} from './geojson-layer/filter-arrow-layer'; export * from './layer-utils/shader-utils'; diff --git a/src/layers/src/arrow-layer/arrow-layer.ts b/src/layers/src/arrow-layer/arrow-layer.ts index ed3c268fcb..79b65f9033 100644 --- a/src/layers/src/arrow-layer/arrow-layer.ts +++ b/src/layers/src/arrow-layer/arrow-layer.ts @@ -28,12 +28,14 @@ import { getBinaryGeometriesFromArrow, parseGeometryFromArrow } from '@kepler.gl/utils'; - +import {FilterArrowExtension} from '@kepler.gl/deckgl-layers'; import GeoJsonLayer, {SUPPORTED_ANALYZER_TYPES} from '../geojson-layer/geojson-layer'; export default class ArrowLayer extends GeoJsonLayer { binaryFeatures: BinaryFeatures[]; dataContainer: DataContainerInterface | null; + filteredIndex: Uint8ClampedArray | null; + filteredIndexTrigger: number[]; // constructor constructor(props) { @@ -41,6 +43,8 @@ export default class ArrowLayer extends GeoJsonLayer { this.dataContainer = null; this.binaryFeatures = []; + this.filteredIndex = null; + this.filteredIndexTrigger = []; } static findDefaultLayerProps({label, fields = []}: KeplerTable) { @@ -84,6 +88,17 @@ export default class ArrowLayer extends GeoJsonLayer { calculateDataAttribute({dataContainer, filteredIndex}, getPosition) { // TODO: filter arrow table using predicate // filteredIndex.map(i => this.dataToFeature[i]).filter(d => d); + // filter arrow table by values and make a partial copy of the raw table could be expensive + // so we will use filteredIndex to create an attribute e.g. filtered (bool) for deck.gl layer + if (!this.filteredIndex) { + this.filteredIndex = new Uint8ClampedArray(dataContainer.numRows()); + } + this.filteredIndex.fill(0); + for (let i = 0; i < filteredIndex.length; ++i) { + this.filteredIndex[filteredIndex[i]] = 1; + } + + this.filteredIndexTrigger = filteredIndex; this.dataContainer = dataContainer; return this.binaryFeatures; } @@ -99,7 +114,7 @@ export default class ArrowLayer extends GeoJsonLayer { // deck.gl geojson-layer will use binaryToFeatureForAccesor(data, index) // to get feature from binary data, and the properties of the feature - // {index: i} can be used for gpu filter + // e.g. properties: {index: i} can be used for gpu filter const customFilterValueAccessor = (dc, d, fieldIndex) => { return dc.valueAt(d.properties.index, fieldIndex); }; @@ -110,6 +125,11 @@ export default class ArrowLayer extends GeoJsonLayer { const dataAccessor = dc => d => { return {index: d.properties.index}; }; + + const isFilteredAccessor = d => { + return this.filteredIndex ? this.filteredIndex[d.properties.index] : 1; + }; + const accessors = this.getAttributeAccessors({dataAccessor, dataContainer}); // return layerData @@ -119,6 +139,7 @@ export default class ArrowLayer extends GeoJsonLayer { indexAccessor, customFilterValueAccessor ), + getFiltered: isFilteredAccessor, ...accessors }; } @@ -132,6 +153,7 @@ export default class ArrowLayer extends GeoJsonLayer { const {binaryGeometries, bounds, featureTypes} = getBinaryGeometriesFromArrow(geoColumn); this.binaryFeatures = binaryGeometries; + // since there is no feature.properties.radius, we set fixedRadius to false const fixedRadius = false; this.updateMeta({bounds, fixedRadius, featureTypes}); } @@ -195,7 +217,8 @@ export default class ArrowLayer extends GeoJsonLayer { const updateTriggers = { ...this.getVisualChannelUpdateTriggers(), - getFilterValue: gpuFilter.filterValueUpdateTriggers + getFilterValue: gpuFilter.filterValueUpdateTriggers, + getFiltered: this.filteredIndexTrigger }; const defaultLayerProps = this.getDefaultDeckLayerProps(opts); @@ -225,6 +248,7 @@ export default class ArrowLayer extends GeoJsonLayer { capRounded: true, jointRounded: true, updateTriggers, + extensions: [...defaultLayerProps.extensions, new FilterArrowExtension()], _subLayerProps: { ...(featureTypes?.polygon ? {'polygons-stroke': opaOverwrite} : {}), ...(featureTypes?.line ? {linestrings: opaOverwrite} : {}), diff --git a/src/utils/src/arrow-utils.ts b/src/utils/src/arrow-utils.ts index aa52b65612..c932610d4e 100644 --- a/src/utils/src/arrow-utils.ts +++ b/src/utils/src/arrow-utils.ts @@ -82,7 +82,7 @@ function updateBoundsFromGeoArrowSamples( flatCoords: Float64Array, nDim: number, bounds: [number, number, number, number], - sampleSize: number = 1000 + sampleSize: number = 100 ) { const numberOfFeatures = flatCoords.length / nDim; const sampleStep = Math.max(Math.floor(numberOfFeatures / sampleSize), 1);