add gpu filter

src/deckgl-layers/src/geojson-layer/filter-arrow-layer.ts

@@ -0,0 +1,50 @@
+import {Layer, LayerExtension} from '@deck.gl/core';
+import {LayerContext} from '@deck.gl/core/lib/layer';
+import GL from '@luma.gl/constants';
+
+import shaderModule from './filter-shader-module';
+
+const defaultProps = {
+  getFiltered: {type: 'accessor', value: 1}
+};
+
+export type FilterArrowExtensionProps = {
+  getFiltered?: () => number;
+};
+
+// Write an extension to filter arrow layer:
+// an instanced attribute 'instanceFiltered' is added to the layer to indicate whether the feature has been Filtered
+// the shader module is modified to discard the feature if instanceFiltered is 0
+// the accessor getFiltered is used to get the value of instanceFiltered based on filteredIndex in Arrowlayer
+export default class FilterArrowExtension extends LayerExtension {
+  static defaultProps = defaultProps;
+  static extensionName = 'FilterArrowExtension';
+
+  getShaders(extension: any) {
+    return {
+      modules: [shaderModule],
+      defines: {}
+    };
+  }
+
+  initializeState(this: Layer<FilterArrowExtensionProps>, context: LayerContext, extension: this) {
+    const attributeManager = this.getAttributeManager();
+    if (attributeManager) {
+      attributeManager.add({
+        filtered: {
+          size: 1,
+          type: GL.FLOAT,
+          accessor: 'getFiltered',
+          shaderAttributes: {
+            filtered: {
+              divisor: 0
+            },
+            instanceFiltered: {
+              divisor: 1
+            }
+          }
+        }
+      });
+    }
+  }
+}

src/deckgl-layers/src/geojson-layer/filter-shader-module.ts

@@ -0,0 +1,39 @@
+import {project} from '@deck.gl/core';
+
+const vs = `
+  #ifdef NON_INSTANCED_MODEL
+    #define FILTER_ARROW_ATTRIB filtered
+  #else
+    #define FILTER_ARROW_ATTRIB instanceFiltered
+  #endif
+  attribute float FILTER_ARROW_ATTRIB;
+`;
+
+const fs = ``;
+
+const inject = {
+  'vs:#decl': `
+    varying float is_filtered;
+  `,
+  'vs:#main-end': `
+    is_filtered = FILTER_ARROW_ATTRIB;
+  `,
+  'fs:#decl': `
+    varying float is_filtered;
+  `,
+  'fs:DECKGL_FILTER_COLOR': `
+    // abandon the fragments if  it is not filtered
+    if (is_filtered == 0.) {
+      discard;
+    }
+  `
+};
+
+export default {
+  name: 'filter-arrow',
+  dependencies: [project],
+  vs: vs,
+  fs: fs,
+  inject: inject,
+  getUniforms: () => {}
+}

src/deckgl-layers/src/index.ts

@@ -9,6 +9,7 @@ export {default as EnhancedGridLayer} from './grid-layer/enhanced-cpu-grid-layer
 export {default as EnhancedHexagonLayer} from './hexagon-layer/enhanced-hexagon-layer';
 export {default as EnhancedLineLayer} from './line-layer/line-layer';
 export {default as SvgIconLayer} from './svg-icon-layer/svg-icon-layer';
+export {default as FilterArrowExtension} from './geojson-layer/filter-arrow-layer';
 
 export * from './layer-utils/shader-utils';
 

src/layers/src/arrow-layer/arrow-layer.ts

@@ -28,19 +28,23 @@ import {
   getBinaryGeometriesFromArrow,
   parseGeometryFromArrow
 } from '@kepler.gl/utils';
-
+import {FilterArrowExtension} from '@kepler.gl/deckgl-layers';
 import GeoJsonLayer, {SUPPORTED_ANALYZER_TYPES} from '../geojson-layer/geojson-layer';
 
 export default class ArrowLayer extends GeoJsonLayer {
   binaryFeatures: BinaryFeatures[];
   dataContainer: DataContainerInterface | null;
+  filteredIndex: Uint8ClampedArray | null;
+  filteredIndexTrigger: number[];
 
   // constructor
   constructor(props) {
     super(props);
 
     this.dataContainer = null;
     this.binaryFeatures = [];
+    this.filteredIndex = null;
+    this.filteredIndexTrigger = [];
   }
 
   static findDefaultLayerProps({label, fields = []}: KeplerTable) {
@@ -84,6 +88,17 @@ export default class ArrowLayer extends GeoJsonLayer {
   calculateDataAttribute({dataContainer, filteredIndex}, getPosition) {
     // TODO: filter arrow table using predicate
     // filteredIndex.map(i => this.dataToFeature[i]).filter(d => d);
+    // filter arrow table by values and make a partial copy of the raw table could be expensive
+    // so we will use filteredIndex to create an attribute e.g. filtered (bool) for deck.gl layer
+    if (!this.filteredIndex) {
+      this.filteredIndex = new Uint8ClampedArray(dataContainer.numRows());
+    }
+    this.filteredIndex.fill(0);
+    for (let i = 0; i < filteredIndex.length; ++i) {
+      this.filteredIndex[filteredIndex[i]] = 1;
+    }
+
+    this.filteredIndexTrigger = filteredIndex;
     this.dataContainer = dataContainer;
     return this.binaryFeatures;
   }
@@ -99,7 +114,7 @@ export default class ArrowLayer extends GeoJsonLayer {
 
     // deck.gl geojson-layer will use binaryToFeatureForAccesor(data, index)
     // to get feature from binary data, and the properties of the feature
-    // {index: i} can be used for gpu filter
+    // e.g. properties: {index: i} can be used for gpu filter
     const customFilterValueAccessor = (dc, d, fieldIndex) => {
       return dc.valueAt(d.properties.index, fieldIndex);
     };
@@ -110,6 +125,11 @@ export default class ArrowLayer extends GeoJsonLayer {
     const dataAccessor = dc => d => {
       return {index: d.properties.index};
     };
+
+    const isFilteredAccessor = d => {
+      return this.filteredIndex ? this.filteredIndex[d.properties.index] : 1;
+    };
+
     const accessors = this.getAttributeAccessors({dataAccessor, dataContainer});
 
     // return layerData
@@ -119,6 +139,7 @@ export default class ArrowLayer extends GeoJsonLayer {
         indexAccessor,
         customFilterValueAccessor
       ),
+      getFiltered: isFilteredAccessor,
       ...accessors
     };
   }
@@ -132,6 +153,7 @@ export default class ArrowLayer extends GeoJsonLayer {
     const {binaryGeometries, bounds, featureTypes} = getBinaryGeometriesFromArrow(geoColumn);
     this.binaryFeatures = binaryGeometries;
 
+    // since there is no feature.properties.radius, we set fixedRadius to false
     const fixedRadius = false;
     this.updateMeta({bounds, fixedRadius, featureTypes});
   }
@@ -195,7 +217,8 @@ export default class ArrowLayer extends GeoJsonLayer {
 
     const updateTriggers = {
       ...this.getVisualChannelUpdateTriggers(),
-      getFilterValue: gpuFilter.filterValueUpdateTriggers
+      getFilterValue: gpuFilter.filterValueUpdateTriggers,
+      getFiltered: this.filteredIndexTrigger
     };
 
     const defaultLayerProps = this.getDefaultDeckLayerProps(opts);
@@ -225,6 +248,7 @@ export default class ArrowLayer extends GeoJsonLayer {
         capRounded: true,
         jointRounded: true,
         updateTriggers,
+        extensions: [...defaultLayerProps.extensions, new FilterArrowExtension()],
         _subLayerProps: {
           ...(featureTypes?.polygon ? {'polygons-stroke': opaOverwrite} : {}),
           ...(featureTypes?.line ? {linestrings: opaOverwrite} : {}),

src/utils/src/arrow-utils.ts

@@ -82,7 +82,7 @@ function updateBoundsFromGeoArrowSamples(
   flatCoords: Float64Array,
   nDim: number,
   bounds: [number, number, number, number],
-  sampleSize: number = 1000
+  sampleSize: number = 100
 ) {
   const numberOfFeatures = flatCoords.length / nDim;
   const sampleStep = Math.max(Math.floor(numberOfFeatures / sampleSize), 1);