wip

src/layers/package.json

@@ -48,17 +48,20 @@
     "@loaders.gl/wkt": "^3.0.9",
     "@luma.gl/constants": "^8.5.10",
     "@mapbox/geojson-normalize": "0.0.1",
-    "@nebula.gl/layers": "1.0.2-alpha.1",
     "@nebula.gl/edit-modes": "1.0.2-alpha.1",
+    "@nebula.gl/layers": "1.0.2-alpha.1",
     "@turf/bbox": "^6.0.1",
     "@turf/boolean-within": "^6.0.1",
+    "@turf/center": "^6.1.4",
+    "@turf/center-of-mass": "^6.1.4",
     "@turf/helpers": "^6.1.4",
     "@types/geojson": "^7946.0.7",
     "@types/keymirror": "^0.1.1",
     "@types/lodash.memoize": "^4.1.7",
     "@types/lodash.uniq": "^4.5.7",
     "@types/styled-components": "^5.1.25",
     "d3-shape": "^1.2.0",
+    "flatbush": "^4.2.0",
     "global": "^4.3.0",
     "keymirror": "^0.1.1",
     "lodash.memoize": "^4.1.2",

src/layers/src/geojson-layer/geojson-layer.ts

@@ -18,6 +18,7 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 // THE SOFTWARE.
 
+import Flatbush from 'flatbush';
 import uniq from 'lodash.uniq';
 import {DATA_TYPES} from 'type-analyzer';
 import {Feature, Polygon} from 'geojson';
@@ -189,6 +190,7 @@ export default class GeoJsonLayer extends Layer {
   declare meta: GeoJsonLayerMeta;
   dataToFeature: GeojsonDataMaps;
   centroids: number[][];
+  static index = new Map<string, Flatbush>();
 
   constructor(props) {
     super(props);
@@ -365,9 +367,22 @@ export default class GeoJsonLayer extends Layer {
     };
   }
 
+  getSpatialIndex() {
+    if (!GeoJsonLayer.index.get(this.id) && this.centroids.length > 0) {
+      const index = new Flatbush(this.centroids.length);
+      this.centroids.forEach(c => index?.add(c[0], c[1], c[0], c[1]));
+      index.finish();
+      GeoJsonLayer.index.set(this.id, index);
+    }
+    return GeoJsonLayer.index.get(this.id);
+  }
+
   getCentroids(): number[][] {
     if (this.centroids.length === 0) {
       this.centroids = getGeojsonMeanCenters(this.dataToFeature);
+      console.time('build spatial index');
+      this.getSpatialIndex();
+      console.timeEnd('build spatial index');
     }
     return this.centroids;
   }
@@ -376,6 +391,8 @@ export default class GeoJsonLayer extends Layer {
     if (this.centroids.length === 0 || !this.centroids[index]) {
       return false;
     }
+    // check if index is in existed spatial index query
+
     const isReactangle = polygon.properties?.shape === 'Rectangle';
     const point = this.centroids[index];
     // check if point is in polygon
@@ -390,7 +407,6 @@ export default class GeoJsonLayer extends Layer {
     const getFeature = this.getPositionAccessor(dataContainer);
     this.dataToFeature = getGeojsonDataMaps(dataContainer, getFeature);
     this.centroids = this.getCentroids();
-
     // get bounds from features
     const bounds = getGeojsonBounds(this.dataToFeature);
     // if any of the feature has properties.radius set to be true

src/layers/src/geojson-layer/geojson-utils.ts

@@ -20,6 +20,8 @@
 
 import normalize from '@mapbox/geojson-normalize';
 import bbox from '@turf/bbox';
+// import center from '@turf/center';
+// import centerOfMass from '@turf/center-of-mass';
 import {parseSync} from '@loaders.gl/core';
 import {WKBLoader, WKTLoader} from '@loaders.gl/wkt';
 import {binaryToGeometry} from '@loaders.gl/gis';
@@ -177,16 +179,20 @@ function getMeanCenterFromGeometries(geometries: Geometry[]): number[] {
  * @returns {number[][]} [[lng, lat], ...]
  */
 export function getGeojsonMeanCenters(dataToFeature: GeojsonDataMaps): number[][] {
+  console.time('getGeojsonMeanCenters');
   const meanCenters: number[][] = [];
   for (let i = 0; i < dataToFeature.length; i++) {
     const feature = dataToFeature[i];
     if (feature) {
+      // TODO: use line interpolate to get center of line for LineString
+      // meanCenters.push(centerOfMass(feature).geometry.coordinates);
       const geometries = feature.geometry.type === 'GeometryCollection' ?
         feature.geometry.geometries : [feature.geometry];
       const center = getMeanCenterFromGeometries(geometries);
       meanCenters.push(center);
     }
   }
+  console.timeEnd('getGeojsonMeanCenters');
   return meanCenters;
 }
 

src/table/src/kepler-table.ts

@@ -20,13 +20,14 @@
 
 import {console as Console} from 'global/console';
 import {ascending, descending} from 'd3-array';
-
+import {intersection} from 'lodash';
 import {
   TRIP_POINT_FIELDS,
   SORT_ORDER,
   ALL_FIELD_TYPES,
   ALTITUDE_FIELDS,
-  SCALE_TYPES
+  SCALE_TYPES,
+  FILTER_TYPES
 } from '@kepler.gl/constants';
 import {
   RGBColor,
@@ -41,7 +42,7 @@ import {
 
 import {getGpuFilterProps, getDatasetFieldIndexForFilter} from './gpu-filter-utils';
 
-import {Layer} from '@kepler.gl/layers';
+import {Layer, KeplerGlLayers} from '@kepler.gl/layers';
 import {
   generateHashId,
   getSortingFunction,
@@ -59,7 +60,8 @@ import {
   getLogDomain,
   getOrdinalDomain,
   getQuantileDomain,
-  DataContainerInterface
+  DataContainerInterface,
+  createIndexedDataContainer
 } from '@kepler.gl/utils';
 
 export type GpuFilter = {
@@ -315,30 +317,51 @@ class KeplerTable {
     const shouldCalIndex = Boolean(this.changedFilters.cpu);
 
     let filterResult: FilterResult = {};
+    console.time('Filter');
     if (shouldCalDomain || shouldCalIndex) {
       const dynamicDomainFilters = shouldCalDomain ? filterRecord.dynamicDomain : null;
       const cpuFilters = shouldCalIndex ? filterRecord.cpu : null;
 
+      // use layer index to build IndexedDataContainer to narrow down the data
+      console.time('filterDataByFilterTypes');
+      const preFilteredIndexes: number[][] = [];
+      for (const filter of filters) {
+        if (filter.type === FILTER_TYPES.polygon) {
+          const filteredLayers = layers.filter(l => l.config.dataId === this.id);
+          // iterator over filteredLayers, and use it's spatial index to filter data
+          filteredLayers?.forEach(layer => {
+            const index = KeplerGlLayers.GeojsonLayer.index.get(layer.id);
+            const [minX, minY, maxX, maxY] = filter.value.properties.bbox;
+            const foundIndexes = index?.search(minX, minY, maxX, maxY) || [];
+            preFilteredIndexes.push(foundIndexes);
+          });
+        }
+      }
+      console.timeEnd('filterDataByFilterTypes');
+      const filteredDataContainer =
+        preFilteredIndexes.length > 0
+          ? createIndexedDataContainer(dataContainer, intersection(preFilteredIndexes.flat()))
+          : dataContainer;
+
       const filterFuncs = filters.reduce((acc, filter) => {
         const fieldIndex = getDatasetFieldIndexForFilter(this.id, filter);
         const field = fieldIndex !== -1 ? fields[fieldIndex] : null;
-
         return {
           ...acc,
-          [filter.id]: getFilterFunction(field, this.id, filter, layers, dataContainer)
+          [filter.id]: getFilterFunction(field, this.id, filter, layers, filteredDataContainer)
         };
       }, {});
 
-      // TODO apply index to narrow down the filter range
       filterResult = filterDataByFilterTypes(
         {dynamicDomainFilters, cpuFilters, filterFuncs},
-        dataContainer
+        filteredDataContainer
       );
     }
 
     this.filteredIndex = filterResult.filteredIndex || this.filteredIndex;
     this.filteredIndexForDomain =
       filterResult.filteredIndexForDomain || this.filteredIndexForDomain;
+    console.timeEnd('Filter');
 
     return this;
   }

src/utils/package.json

@@ -32,8 +32,8 @@
   "dependencies": {
     "@kepler.gl/constants": "3.0.0-alpha.0",
     "@kepler.gl/types": "3.0.0-alpha.0",
-    "@loaders.gl/wkt": "3.0.9",
     "@loaders.gl/gis": "3.0.9",
+    "@loaders.gl/wkt": "3.0.9",
     "@luma.gl/constants": "^8.5.10",
     "@luma.gl/core": "^8.5.10",
     "@mapbox/geo-viewport": "^0.4.1",

src/utils/src/filter-utils.ts

@@ -456,6 +456,9 @@ export const getPolygonFilterFunctor = (layer, filter, dataContainer) => {
       };
     case LAYER_TYPES.geojson:
       return data => {
+        if (filter.value.properties?.shape === 'Rectangle') {
+          return true;
+        }
         if (layer.isInPolygon) {
           return layer.isInPolygon(data, data.index, filter.value);
         }
@@ -553,9 +556,10 @@ export function filterDataByFilterTypes(
   const filterFuncCaller = (filter: Filter) => filterFuncs[filter.id](filterContext);
 
   const numRows = dataContainer.numRows();
+  const plainIndexes = dataContainer.getPlainIndex();
   if (dynamicDomainFilters) {
     for (let i = 0; i < numRows; ++i) {
-      filterContext.index = i;
+      filterContext.index = plainIndexes[i];
 
       const matchForDomain = dynamicDomainFilters && dynamicDomainFilters.every(filterFuncCaller);
       if (matchForDomain) {
@@ -566,7 +570,7 @@ export function filterDataByFilterTypes(
   // TODO: with a index, we should be able to avoid iterate through all data
   if (cpuFilters) {
     for (let i = 0; i < numRows; ++i) {
-      filterContext.index = i;
+      filterContext.index = plainIndexes[i];
       const matchForRender = cpuFilters && cpuFilters.every(filterFuncCaller);
       if (matchForRender) {
         filteredIndex.push(filterContext.index);

src/utils/src/indexed-data-container.ts

@@ -110,7 +110,7 @@ export class IndexedDataContainer implements DataContainerInterface {
   }
 
   getPlainIndex(): number[] {
-    return this._indices.map((_, i) => i);
+    return this._indices;
   }
 
   flattenData(): any[][] {