From 7af4f241eea3033a9608dae73ca436a501132309 Mon Sep 17 00:00:00 2001 From: Xun Li Date: Thu, 21 Sep 2023 15:25:16 -0700 Subject: [PATCH] wip --- src/layers/package.json | 5 ++- src/layers/src/geojson-layer/geojson-layer.ts | 18 ++++++++- src/layers/src/geojson-layer/geojson-utils.ts | 6 +++ src/table/src/kepler-table.ts | 39 +++++++++++++++---- src/utils/package.json | 2 +- src/utils/src/filter-utils.ts | 8 +++- src/utils/src/indexed-data-container.ts | 2 +- 7 files changed, 66 insertions(+), 14 deletions(-) diff --git a/src/layers/package.json b/src/layers/package.json index 2c0c5607c0..e8dd5fb1ef 100644 --- a/src/layers/package.json +++ b/src/layers/package.json @@ -48,10 +48,12 @@ "@loaders.gl/wkt": "^3.0.9", "@luma.gl/constants": "^8.5.10", "@mapbox/geojson-normalize": "0.0.1", - "@nebula.gl/layers": "1.0.2-alpha.1", "@nebula.gl/edit-modes": "1.0.2-alpha.1", + "@nebula.gl/layers": "1.0.2-alpha.1", "@turf/bbox": "^6.0.1", "@turf/boolean-within": "^6.0.1", + "@turf/center": "^6.1.4", + "@turf/center-of-mass": "^6.1.4", "@turf/helpers": "^6.1.4", "@types/geojson": "^7946.0.7", "@types/keymirror": "^0.1.1", @@ -59,6 +61,7 @@ "@types/lodash.uniq": "^4.5.7", "@types/styled-components": "^5.1.25", "d3-shape": "^1.2.0", + "flatbush": "^4.2.0", "global": "^4.3.0", "keymirror": "^0.1.1", "lodash.memoize": "^4.1.2", diff --git a/src/layers/src/geojson-layer/geojson-layer.ts b/src/layers/src/geojson-layer/geojson-layer.ts index 0521f65c9d..cd17b794ae 100644 --- a/src/layers/src/geojson-layer/geojson-layer.ts +++ b/src/layers/src/geojson-layer/geojson-layer.ts @@ -18,6 +18,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +import Flatbush from 'flatbush'; import uniq from 'lodash.uniq'; import {DATA_TYPES} from 'type-analyzer'; import {Feature, Polygon} from 'geojson'; @@ -189,6 +190,7 @@ export default class GeoJsonLayer extends Layer { declare meta: GeoJsonLayerMeta; dataToFeature: GeojsonDataMaps; centroids: number[][]; + static index = new Map(); constructor(props) { super(props); @@ -365,9 +367,22 @@ export default class GeoJsonLayer extends Layer { }; } + getSpatialIndex() { + if (!GeoJsonLayer.index.get(this.id) && this.centroids.length > 0) { + const index = new Flatbush(this.centroids.length); + this.centroids.forEach(c => index?.add(c[0], c[1], c[0], c[1])); + index.finish(); + GeoJsonLayer.index.set(this.id, index); + } + return GeoJsonLayer.index.get(this.id); + } + getCentroids(): number[][] { if (this.centroids.length === 0) { this.centroids = getGeojsonMeanCenters(this.dataToFeature); + console.time('build spatial index'); + this.getSpatialIndex(); + console.timeEnd('build spatial index'); } return this.centroids; } @@ -376,6 +391,8 @@ export default class GeoJsonLayer extends Layer { if (this.centroids.length === 0 || !this.centroids[index]) { return false; } + // check if index is in existed spatial index query + const isReactangle = polygon.properties?.shape === 'Rectangle'; const point = this.centroids[index]; // check if point is in polygon @@ -390,7 +407,6 @@ export default class GeoJsonLayer extends Layer { const getFeature = this.getPositionAccessor(dataContainer); this.dataToFeature = getGeojsonDataMaps(dataContainer, getFeature); this.centroids = this.getCentroids(); - // get bounds from features const bounds = getGeojsonBounds(this.dataToFeature); // if any of the feature has properties.radius set to be true diff --git a/src/layers/src/geojson-layer/geojson-utils.ts b/src/layers/src/geojson-layer/geojson-utils.ts index 8170d306d2..27213a3f97 100644 --- a/src/layers/src/geojson-layer/geojson-utils.ts +++ b/src/layers/src/geojson-layer/geojson-utils.ts @@ -20,6 +20,8 @@ import normalize from '@mapbox/geojson-normalize'; import bbox from '@turf/bbox'; +// import center from '@turf/center'; +// import centerOfMass from '@turf/center-of-mass'; import {parseSync} from '@loaders.gl/core'; import {WKBLoader, WKTLoader} from '@loaders.gl/wkt'; import {binaryToGeometry} from '@loaders.gl/gis'; @@ -177,16 +179,20 @@ function getMeanCenterFromGeometries(geometries: Geometry[]): number[] { * @returns {number[][]} [[lng, lat], ...] */ export function getGeojsonMeanCenters(dataToFeature: GeojsonDataMaps): number[][] { + console.time('getGeojsonMeanCenters'); const meanCenters: number[][] = []; for (let i = 0; i < dataToFeature.length; i++) { const feature = dataToFeature[i]; if (feature) { + // TODO: use line interpolate to get center of line for LineString + // meanCenters.push(centerOfMass(feature).geometry.coordinates); const geometries = feature.geometry.type === 'GeometryCollection' ? feature.geometry.geometries : [feature.geometry]; const center = getMeanCenterFromGeometries(geometries); meanCenters.push(center); } } + console.timeEnd('getGeojsonMeanCenters'); return meanCenters; } diff --git a/src/table/src/kepler-table.ts b/src/table/src/kepler-table.ts index 09b694598d..b169b0160e 100644 --- a/src/table/src/kepler-table.ts +++ b/src/table/src/kepler-table.ts @@ -20,13 +20,14 @@ import {console as Console} from 'global/console'; import {ascending, descending} from 'd3-array'; - +import {intersection} from 'lodash'; import { TRIP_POINT_FIELDS, SORT_ORDER, ALL_FIELD_TYPES, ALTITUDE_FIELDS, - SCALE_TYPES + SCALE_TYPES, + FILTER_TYPES } from '@kepler.gl/constants'; import { RGBColor, @@ -41,7 +42,7 @@ import { import {getGpuFilterProps, getDatasetFieldIndexForFilter} from './gpu-filter-utils'; -import {Layer} from '@kepler.gl/layers'; +import {Layer, KeplerGlLayers} from '@kepler.gl/layers'; import { generateHashId, getSortingFunction, @@ -59,7 +60,8 @@ import { getLogDomain, getOrdinalDomain, getQuantileDomain, - DataContainerInterface + DataContainerInterface, + createIndexedDataContainer } from '@kepler.gl/utils'; export type GpuFilter = { @@ -315,30 +317,51 @@ class KeplerTable { const shouldCalIndex = Boolean(this.changedFilters.cpu); let filterResult: FilterResult = {}; + console.time('Filter'); if (shouldCalDomain || shouldCalIndex) { const dynamicDomainFilters = shouldCalDomain ? filterRecord.dynamicDomain : null; const cpuFilters = shouldCalIndex ? filterRecord.cpu : null; + // use layer index to build IndexedDataContainer to narrow down the data + console.time('filterDataByFilterTypes'); + const preFilteredIndexes: number[][] = []; + for (const filter of filters) { + if (filter.type === FILTER_TYPES.polygon) { + const filteredLayers = layers.filter(l => l.config.dataId === this.id); + // iterator over filteredLayers, and use it's spatial index to filter data + filteredLayers?.forEach(layer => { + const index = KeplerGlLayers.GeojsonLayer.index.get(layer.id); + const [minX, minY, maxX, maxY] = filter.value.properties.bbox; + const foundIndexes = index?.search(minX, minY, maxX, maxY) || []; + preFilteredIndexes.push(foundIndexes); + }); + } + } + console.timeEnd('filterDataByFilterTypes'); + const filteredDataContainer = + preFilteredIndexes.length > 0 + ? createIndexedDataContainer(dataContainer, intersection(preFilteredIndexes.flat())) + : dataContainer; + const filterFuncs = filters.reduce((acc, filter) => { const fieldIndex = getDatasetFieldIndexForFilter(this.id, filter); const field = fieldIndex !== -1 ? fields[fieldIndex] : null; - return { ...acc, - [filter.id]: getFilterFunction(field, this.id, filter, layers, dataContainer) + [filter.id]: getFilterFunction(field, this.id, filter, layers, filteredDataContainer) }; }, {}); - // TODO apply index to narrow down the filter range filterResult = filterDataByFilterTypes( {dynamicDomainFilters, cpuFilters, filterFuncs}, - dataContainer + filteredDataContainer ); } this.filteredIndex = filterResult.filteredIndex || this.filteredIndex; this.filteredIndexForDomain = filterResult.filteredIndexForDomain || this.filteredIndexForDomain; + console.timeEnd('Filter'); return this; } diff --git a/src/utils/package.json b/src/utils/package.json index 90e9c6ed0f..ee270e8ce2 100644 --- a/src/utils/package.json +++ b/src/utils/package.json @@ -32,8 +32,8 @@ "dependencies": { "@kepler.gl/constants": "3.0.0-alpha.0", "@kepler.gl/types": "3.0.0-alpha.0", - "@loaders.gl/wkt": "3.0.9", "@loaders.gl/gis": "3.0.9", + "@loaders.gl/wkt": "3.0.9", "@luma.gl/constants": "^8.5.10", "@luma.gl/core": "^8.5.10", "@mapbox/geo-viewport": "^0.4.1", diff --git a/src/utils/src/filter-utils.ts b/src/utils/src/filter-utils.ts index 993aa664e5..f3c2ac94ca 100644 --- a/src/utils/src/filter-utils.ts +++ b/src/utils/src/filter-utils.ts @@ -456,6 +456,9 @@ export const getPolygonFilterFunctor = (layer, filter, dataContainer) => { }; case LAYER_TYPES.geojson: return data => { + if (filter.value.properties?.shape === 'Rectangle') { + return true; + } if (layer.isInPolygon) { return layer.isInPolygon(data, data.index, filter.value); } @@ -553,9 +556,10 @@ export function filterDataByFilterTypes( const filterFuncCaller = (filter: Filter) => filterFuncs[filter.id](filterContext); const numRows = dataContainer.numRows(); + const plainIndexes = dataContainer.getPlainIndex(); if (dynamicDomainFilters) { for (let i = 0; i < numRows; ++i) { - filterContext.index = i; + filterContext.index = plainIndexes[i]; const matchForDomain = dynamicDomainFilters && dynamicDomainFilters.every(filterFuncCaller); if (matchForDomain) { @@ -566,7 +570,7 @@ export function filterDataByFilterTypes( // TODO: with a index, we should be able to avoid iterate through all data if (cpuFilters) { for (let i = 0; i < numRows; ++i) { - filterContext.index = i; + filterContext.index = plainIndexes[i]; const matchForRender = cpuFilters && cpuFilters.every(filterFuncCaller); if (matchForRender) { filteredIndex.push(filterContext.index); diff --git a/src/utils/src/indexed-data-container.ts b/src/utils/src/indexed-data-container.ts index d2a6f09e6f..96a75bf112 100644 --- a/src/utils/src/indexed-data-container.ts +++ b/src/utils/src/indexed-data-container.ts @@ -110,7 +110,7 @@ export class IndexedDataContainer implements DataContainerInterface { } getPlainIndex(): number[] { - return this._indices.map((_, i) => i); + return this._indices; } flattenData(): any[][] {